Changeset 1556

Timestamp:

05/10/21 11:21:12 (3 years ago)

Author:

davidb

Message:

First cut at supporting time-stretch

File:

• trunk/src/org/apollo/widgets/SampledTrack.java (modified) (6 diffs)

trunk/src/org/apollo/widgets/SampledTrack.java

@@ -15 +15 @@
 import java.awt.geom.Rectangle2D;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.util.LinkedList;
@@ -21 +23 @@
 
 import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.AudioFormat.Encoding;
+import javax.sound.sampled.AudioInputStream;
 import javax.sound.sampled.LineUnavailableException;
 import javax.sound.sampled.UnsupportedAudioFileException;
@@ -48 +52 @@
 import org.apollo.io.AudioPathManager;
 import org.apollo.io.IconRepository;
+import org.apollo.io.LoadedAudioData;
 import org.apollo.items.EmulatedTextItem;
 import org.apollo.items.EmulatedTextItem.TextChangeListener;
@@ -80 +85 @@
 import org.expeditee.items.widgets.Widget;
 
+
+import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.AudioSystem;
+import javax.sound.sampled.LineUnavailableException;
+import javax.sound.sampled.UnsupportedAudioFileException;
+
+import be.tarsos.dsp.AudioDispatcher;
+import be.tarsos.dsp.GainProcessor;
+import be.tarsos.dsp.WaveformSimilarityBasedOverlapAdd;
+import be.tarsos.dsp.WaveformSimilarityBasedOverlapAdd.Parameters;
+import be.tarsos.dsp.io.TarsosDSPAudioFormat;
+import be.tarsos.dsp.io.UniversalAudioInputStream;
+import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
+import be.tarsos.dsp.io.jvm.AudioPlayer;
+import be.tarsos.dsp.io.jvm.WaveformWriter;
+
 /**
  * The sampled track widgets in apollo.
@@ -2072 +2093 @@
                 }
 
+                // Adapted from:
+                //   https://stackoverflow.com/questions/50631179/converting-stereo-to-mono-using-tarsosdsp-does-not-work
+                        
+                private  UniversalAudioInputStream convertToMono(AudioInputStream sourceStream) 
+                {
+                        AudioInputStream targetStream = null;
+                        
+                    AudioFormat sourceFormat = sourceStream.getFormat();
+
+                    // is already mono?
+                    /*if(sourceFormat.getChannels() == 1) {
+                        return sourceStream;
+                    }
+*/
+                    if (sourceFormat.getChannels() > 1) {
+                    
+                        AudioFormat targetFormat = new AudioFormat(
+                                        sourceFormat.getEncoding(),
+                                        sourceFormat.getSampleRate(),
+                                        sourceFormat.getSampleSizeInBits(),
+                                        1,
+                                        // this is the important bit, the framesize needs to change as well,
+                                        // for framesize 4, this calculation leads to new framesize 2
+                                        (sourceFormat.getSampleSizeInBits() + 7) / 8,
+                                        sourceFormat.getFrameRate(),
+                                        sourceFormat.isBigEndian());
+
+                        targetStream = AudioSystem.getAudioInputStream(targetFormat, sourceStream);
+                    }
+                    else {
+                        targetStream = sourceStream;
+                    }
+                    
+                    // better to express in terms of targetFormat !!!
+                    boolean is_signed = (sourceFormat.getEncoding() == Encoding.PCM_SIGNED);
+                    
+                        TarsosDSPAudioFormat ts_audioFormat = new TarsosDSPAudioFormat(
+                                          /* sample rate */ sourceFormat.getSampleRate(),
+                                          /* HERE sample size in bits */ sourceFormat.getSampleSizeInBits(),  
+                                          /* number of channels */ 1,
+                                          /* signed/unsigned data */ is_signed,
+                                          /* big-endian byte order */sourceFormat.isBigEndian()
+                                        );
+                        
+                        UniversalAudioInputStream uis = new UniversalAudioInputStream(targetStream, ts_audioFormat);
+                                        
+                    return uis;
+                }
+                
+                private SampledTrackModel timeStretch(SampledTrackModel orig_track_model, double time_stretch_factor) 
+                {
+                        SampledTrackModel stretched_track = null;
+                        
+                        try {
+                                String stretched_local_target = null;
+                                File stretched_target_file = null;
+                                boolean orig_is_already_timestretched = false;
+                                
+                                
+                                String orig_source = orig_track_model.getFilepath();
+                                String orig_local_filename = orig_track_model.getLocalFilename();
+                                
+                                String orig_source_root = orig_source.substring(0,orig_source.lastIndexOf('.'));                
+                                        
+                                if (orig_source_root.matches("-ts[0-9]+(\\.[0-9]+)$")) {
+                                        // already hooked up to a time-stretched file
+                                        stretched_local_target = orig_local_filename;
+                                        stretched_target_file = new File(orig_source);
+                                        
+                                        orig_is_already_timestretched = true;
+                                }
+                        
+                                if (!orig_is_already_timestretched) {
+        
+                                        // Need to make 44.1 kHz mono version (if does not already exist)
+                                        
+                                        
+                                        String orig_local_filename_root = orig_local_filename.substring(0,orig_local_filename.lastIndexOf('.'));
+                                
+                                        String stretched_target = orig_source_root + "-ts" + time_stretch_factor + ".wav";
+                                        stretched_local_target = orig_local_filename_root + "-ts" + time_stretch_factor + ".wav";
+                                        
+                                        System.out.println("***** Checking for existence of target output wav file: " + stretched_target);
+
+                                        stretched_target_file = new File(stretched_target);
+                                        if (!stretched_target_file.exists()) {
+
+                                                File orig_source_file = new File(orig_source);
+                                                //AudioFormat orig_format = AudioSystem.getAudioFileFormat(orig_source_file).getFormat();       
+                                                
+                                                AudioInputStream orig_audio_input_stream = AudioSystem.getAudioInputStream(orig_source_file);
+                                                                                                
+                                                UniversalAudioInputStream mono_audio_input_stream = convertToMono(orig_audio_input_stream);
+                                                TarsosDSPAudioFormat ts_audioFormat = mono_audio_input_stream.getFormat();
+                                                
+                                                //AudioFormat mono_format = mono_audio_input_stream.getFormat();
+                                                
+                                                
+                                                
+                                                
+                                                //double orig_sample_rate = orig_format.getSampleRate();
+                                                //double mono_sample_rate = mono_format.getSampleRate();
+                                                
+                                                
+                                                //https://stackoverflow.com/questions/31365172/tarsosdsp-pitch-detection-from-wav-file-and-the-result-frequency-is-always-les
+                                                /*
+                                                InputStream is = new FileInputStream(orig_source_file);
+                                                
+                                                TarsosDSPAudioFormat ts_audioFormat = new TarsosDSPAudioFormat(
+                                                  / * sample rate * / 44100,
+                                                  / * HERE sample size in bits * / 16,  // used to be 32
+                                                  / * number of channels * / 1,
+                                                  / * signed/unsigned data * / true,
+                                                  / * big-endian byte order * / false
+                                                );
+                                                UniversalAudioInputStream uis = new UniversalAudioInputStream(is, ts_audioFormat);
+                                                
+                                                */
+                                                
+                                                //AudioDispatcher dispatcher = new AudioDispatcher(uis, 2048, 0);
+
+                                                
+                                                
+                                                // Consider allowing finer grained control of WSOLA params through @annotations:
+                                                //  int sequenceMs
+                                                //  int seekWindowMs
+                                                //  int overlapMs
+                                                float ts_sample_rate = ts_audioFormat.getSampleRate();
+                                                Parameters slowdown_params = Parameters.slowdownDefaults(time_stretch_factor,ts_sample_rate);
+                                                WaveformSimilarityBasedOverlapAdd wsola = new WaveformSimilarityBasedOverlapAdd(slowdown_params);
+
+                                                WaveformWriter stretched_target_writer = new WaveformWriter(ts_audioFormat,stretched_target);
+
+                                                //AudioDispatcher dispatcher = AudioDispatcherFactory.fromFile(orig_source_file,wsola.getInputBufferSize(),wsola.getOverlap());
+                                                AudioDispatcher dispatcher = new AudioDispatcher(mono_audio_input_stream,wsola.getInputBufferSize(),wsola.getOverlap());
+                                                wsola.setDispatcher(dispatcher);
+                                                dispatcher.addAudioProcessor(wsola);
+                                                dispatcher.addAudioProcessor(stretched_target_writer);
+                                                System.out.println("Starting TimeStetch with factor: " + time_stretch_factor + "...");
+                                                dispatcher.run();
+                                                System.out.println("... done");
+
+                                                //AudioFormat stretched_target_format = AudioSystem.getAudioFileFormat(stretched_target_file).getFormat();      
+
+                                        }
+                                
+                                        LoadedAudioData stretched_audio_data = AudioIO.loadAudioFile(stretched_target_file, null);
+                                        
+                                        // loading audio data could have been cancelled
+                                        if (stretched_audio_data != null) { 
+                                                // Create the track
+                                                stretched_track = new SampledTrackModel(
+                                                                stretched_audio_data.getAudioBytes(), 
+                                                                stretched_audio_data.getAudioFormat(),
+                                                                stretched_local_target);
+
+                                                stretched_track.setFilepath(stretched_target);
+                                        }
+                                        else {
+                                                System.out.println("Cancelled loading in time-stretched WAV file: " + stretched_target)
+;                                       }
+                                }
+                                
+                        }
+                        catch (Exception e) {
+                                e.printStackTrace();
+                        }
+                        
+                        return stretched_track;
+                }
+                
                 private void playPauseResume() throws LineUnavailableException {
                         if (!SoundDesk.getInstance().isPlaying(trackMix.getChannelID())) { // play / resume
@@ -2113 +2305 @@
                                 
                                 if (startFrame < endFrame) {
+
+                                        // Conditions are right, to play!
+                                        
+                                        /*
+                                        Frame current = DisplayController.getCurrentFrame();
+                                        // Or perhaps (??) we should be getting the current frame via:
+                                        //Frame current = getSource().getParentOrCurrentFrame();
+                                        
+                                        //Text time_stretch_annotation = current.getAnnotation("TimeStretchFactor");
+                                        String time_stretch_annotation_valuestr = current.getAnnotationValue("TimeStretchFactor");
+                                        
+                                        if (time_stretch_annotation_valuestr != null) {
+                                                //String time_stretch_str = time_stretch_annotation.getText();
+                                                //time_stretch_str = time_stretch_str.replaceFirst("^\\s*@time-stretch-factor(:?)\\s*","");
+                                                //System.out.println("**** time stretch str = " + time_stretch_annotation_valuestr);
+                                                
+                                                double time_stretch = Double.parseDouble(time_stretch_annotation_valuestr);
+                                                
+                                                if (time_stretch != 1.0) {
+                                                        SampledTrackModel stretched_track_model = timeStretch(trackModel,time_stretch);
+                                                        if (stretched_track_model != null) {
+                                                                trackModel = stretched_track_model;
+                                                        }
+                                                }
+                                        }
+                                        */
+                                
+                                        
                                         SoundDesk.getInstance().playSampledTrackModel(
                                                         trackModel,