source: trunk/src/org/apollo/gui/WaveFormRenderProccessingUnit.java@ 1561

Last change on this file since 1561 was 1561, checked in by davidb, 3 years ago

A set of changes that spans three things: beat detection, time stretching; and a debug class motivated by the need to look at a canvas redraw issue most notable when a waveform widget is playing
File size: 12.6 KB
Line 
1package org.apollo.gui;
2
3import java.awt.Graphics2D;
4import java.util.LinkedList;
5import java.util.List;
6
7import javax.sound.sampled.AudioFormat;
8
9import org.apollo.audio.ApolloSubjectChangedEvent;
10import org.apollo.mvc.AbstractSubject;
11import org.apollo.mvc.SubjectChangedEvent;
12import org.expeditee.core.Image;
13import org.expeditee.gio.swing.SwingConversions;
14import org.expeditee.gio.swing.SwingMiscManager;
15
16public class WaveFormRenderProccessingUnit {
17
18 /** Limit the amount of render threads. */
19 private RenderThread[] renderThreads = new RenderThread[4];
20
21 private LinkedList<WaveFormRenderTask> taskQueue = new LinkedList<WaveFormRenderTask>();
22
23 private static WaveFormRenderProccessingUnit instance = new WaveFormRenderProccessingUnit(); // single design pattern
24
25 /**
26 * @return The singleton instance of the WaveFormRenderProccessingUnit
27 */
28 public static WaveFormRenderProccessingUnit getInstance() { // single design pattern
29 return instance;
30 }
31
32 private WaveFormRenderProccessingUnit() { // singleton design pattern
33 }
34
35 /**
36 * Queues a task for rendering as soon as possible (asynchronously).
37 *
38 * @param task
39 * The task to queue
40 *
41 * @throws NullPointerException
42 * If task is null.
43 *
44 * @throws IllegalArgumentException
45 * If task has been proccessed before (must not be re-used).
46 * Or if it is already queued for rendering.
47 */
48 public void queueTask(WaveFormRenderTask task) {
49 if (task == null) throw new NullPointerException("task");
50
51 // Ensure not re-using a WaveFormRenderTask
52 if (task.hasStarted()) throw new IllegalArgumentException("task");
53
54 // Add to the queue
55 synchronized (taskQueue) {
56
57 // Check that not already on the queue
58 if (taskQueue.contains(task)) {
59 throw new IllegalArgumentException("task");
60 }
61
62 taskQueue.add(task);
63
64
65 // If there is are dead thread, re-animate it to ensure
66 // the task begins as soon as possible.
67 for (int i = 0; i < renderThreads.length; i++) {
68
69 if (renderThreads[i] == null
70 || renderThreads[i].isFinishing()
71 || !renderThreads[i].isAlive()) {
72
73 renderThreads[i] = new RenderThread(i);
74 renderThreads[i].start();
75 break;
76 }
77
78 }
79
80 }
81
82
83 }
84
85 /**
86 * Cancels a task from rendering.
87 *
88 * @param task
89 * The task to cancel.
90 *
91 * @throws NullPointerException
92 * If task is null.
93 */
94 public void cancelTask(WaveFormRenderTask task) {
95 if (task == null) throw new NullPointerException("task");
96
97 task.keepRendering = false;
98 synchronized (taskQueue) {
99 taskQueue.remove(task);
100 }
101 }
102
103 /**
104 * Render of wave forms is done in a dedicated thread because some graphs may have
105 * to proccess / render a lot of samples depending on the zoom / size of the audio track.
106 *
107 * From a usability perspective, this means that the waveform can be computed accurately
108 * because response time is no longer an issue.
109 *
110 * @author Brook Novak
111 *
112 */
113 private class RenderThread extends Thread {
114
115 private WaveFormRenderTask currentTask;
116
117 private final static float AMPLITUDES_PER_PIXEL = 1.0f; // resolution: how many amplitudes per pixel to render
118 private final static int FRAME_RENDER_RATE = 2000; // The approximate amount of frames to render per pass
119
120 private boolean isFinishing = false;
121
122 public boolean isFinishing() {
123 return isFinishing;
124 }
125
126 /**
127 *
128 * @param audioBytes The reference to the audio bytes to render.
129 *
130 * @param frameLength in frames.
131 */
132 public RenderThread(int id) {
133 super("RenderThread" + id);
134 // Give renderering lowest priority
135 setPriority(Thread.MIN_PRIORITY);
136
137 }
138
139 @Override
140 public void run() {
141
142 while (true) {
143 // Grab a task from the queue
144 synchronized(taskQueue) {
145
146 if (taskQueue.isEmpty()) {
147 isFinishing = true;
148 return;
149 }
150 currentTask = taskQueue.remove();
151 }
152
153 // Perform rendering until the task has been cancelled
154 doRender();
155 }
156 }
157
158 private void doRender() {
159 assert(currentTask != null);
160 assert(!currentTask.hasStarted());
161
162 // Quick check
163 if (!currentTask.keepRendering) return;
164
165 currentTask.setState(WaveFormRenderTask.STATE_RENDERING);
166
167 Graphics2D g = null;
168
169 try {
170
171 int halfMaxHeight;
172 int bufferWidth;
173
174 // Create the graphics and prepare the buffer
175
176 synchronized(currentTask.imageBuffer) {
177
178 halfMaxHeight = currentTask.imageBuffer.getHeight() / 2;
179 bufferWidth = currentTask.imageBuffer.getWidth();
180
181 g = SwingMiscManager.getIfUsingSwingImageManager().getImageGraphics(currentTask.imageBuffer);
182
183 g.setStroke(SwingConversions.toSwingStroke(Strokes.SOLID_1));
184
185 // Clear the buffer with transparent pixels
186 g.setBackground(SwingConversions.toSwingColor(ApolloColorIndexedModels.KEY_COLOR));
187 g.clearRect(0, 0, bufferWidth, currentTask.imageBuffer.getHeight());
188
189 }
190
191 // Render waveforms in chunks - so that can cancel rendering and the
192 // widget can show progress incrementally
193 int lastXPosition = -1;
194 int lastYPosition = -1;
195
196 // Choose how many amplitudes to render in the graph. i.e. waveform resolution
197 int totalAmplitudes = Math.min((int)(bufferWidth * AMPLITUDES_PER_PIXEL), currentTask.frameLength); // get amount of amps to render
198 if (totalAmplitudes == 0) return;
199
200 int currentAmplitude = 0;
201
202 // Calculate the amount of frames to aggregate
203 int aggregationSize = currentTask.frameLength / totalAmplitudes;
204 assert(aggregationSize >= 1);
205
206 // Limit the amount of amplitudes to render (the small chunks) based on the
207 // aggregation size - since it correlation to proccess time.
208 int amplitudeCountPerPass = (FRAME_RENDER_RATE / aggregationSize);
209 if (amplitudeCountPerPass == 0) amplitudeCountPerPass = 1;
210
211 int renderStart;
212 int renderLength;
213
214 // render until finished or cancelled
215 while (currentTask.keepRendering && currentAmplitude < totalAmplitudes) {
216
217 renderStart = currentAmplitude * aggregationSize;
218
219 // At the last pass, render the last remaining bytes
220 renderLength = Math.min(
221 amplitudeCountPerPass * aggregationSize,
222 currentTask.frameLength - renderStart);
223
224 // At the last pass, be sure that the aggregate size does not exeed the render count
225 // so that last samples are not ignored
226 aggregationSize = Math.min(aggregationSize, renderLength);
227
228 // Perform the waveform rendering
229 float[] amps = currentTask.renderer.getSampleAmplitudes(
230 currentTask.audioBytes,
231 currentTask.startFrame + renderStart,
232 renderLength,
233 aggregationSize);
234
235 // Draw the rendered waveform to the buffer
236 synchronized(currentTask.imageBuffer) {
237
238 g.setColor(SwingConversions.toSwingColor(ApolloColorIndexedModels.WAVEFORM_COLOR));
239
240 if (aggregationSize == 1) {
241
242 for (float h : amps) {
243
244 int x = (int)(currentAmplitude * ((float)bufferWidth / (float)(totalAmplitudes - 1)));
245 int y = halfMaxHeight - (int)(h * halfMaxHeight);
246
247 if (currentAmplitude == 0) { // never draw line on first point
248
249 lastXPosition = 0;
250 lastYPosition = y;
251
252 } else {
253
254 g.drawLine(
255 lastXPosition,
256 lastYPosition,
257 x,
258 y);
259
260 lastXPosition = x;
261 lastYPosition = y;
262
263 }
264
265 currentAmplitude ++;
266 }
267
268 } else { // dual version - looks nicer and conceptually easy to see audio
269
270 for (int i = 0; i < amps.length; i+=2) {
271
272 g.setColor(SwingConversions.toSwingColor(ApolloColorIndexedModels.WAVEFORM_COLOR));
273
274 float peak = amps[i];
275 float trough = amps[i + 1];
276
277 int x = (int)(currentAmplitude * ((float)bufferWidth / (float)(totalAmplitudes - 1)));
278 int ypeak = halfMaxHeight - (int)(peak * halfMaxHeight);
279 int ytrough = halfMaxHeight - (int)(trough * halfMaxHeight);
280
281 if (currentAmplitude == 0) { // never draw line on first point
282
283 lastXPosition = lastYPosition = 0;
284
285 } else {
286
287 g.drawLine(
288 x,
289 ypeak,
290 x,
291 ytrough);
292
293 lastXPosition = x;
294 lastYPosition = 0;
295
296 }
297
298 int frameSize = 4;
299 int bi_start = (currentTask.startFrame + renderStart);
300 int bi_end = bi_start + (renderLength);
301
302 for (int bi = bi_start; bi<bi_end; bi++) {
303 if (currentTask.frameIsBeat[bi]) {
304 g.setColor(SwingConversions.toSwingColor(ApolloColorIndexedModels.WAVEFORM_SELECTION_COLOR));
305 g.drawLine(x, halfMaxHeight, x, -halfMaxHeight);
306 break;
307 }
308 }
309
310 currentAmplitude ++;
311 }
312
313 }
314
315 }
316
317
318 } // next pass
319
320 // Lesson learnt: do not request lots of requests to repaint
321 // later on the AWT Event queue otherwise it will get congested
322 // and will freeze up the interact for annoying periods of time.
323 currentTask.setState(WaveFormRenderTask.STATE_STOPPED);
324
325 } finally {
326
327 if (g != null) g.dispose();
328
329 // safety
330 if (!currentTask.isStopped())
331 currentTask.setState(WaveFormRenderTask.STATE_STOPPED);
332
333 currentTask = null;
334 }
335 }
336
337 }
338
339 /**
340 * A task descriptor for rendering wave forms via the WaveFormRenderProccessingUnit.
341 *
342 * Raises {@value ApolloSubjectChangedEvent#RENDER_TASK_INVALIDATION_RECOMENDATION}
343 * when the WaveFormRenderProccessingUnit recommends a refresh
344 * of the BufferedImage. This includes when the WaveFormRenderTask state changes
345 *
346 * The BufferedImage is always locked before handled by the render thread.
347 *
348 * Only can use these once... per render task. I.E. Do not re-use
349 *
350 * @author Brook Novak
351 *
352 */
353 public class WaveFormRenderTask extends AbstractSubject {
354
355 private boolean keepRendering = true; // a cancel flag
356
357 private int state = STATE_PENDING;
358
359 private static final int STATE_PENDING = 1;
360 private static final int STATE_RENDERING = 2;
361 private static final int STATE_STOPPED = 3;
362
363 private Image imageBuffer; // nullified when stopped.
364 private byte[] audioBytes; // nullified when stopped. - Arrays (not contents) are immutable so no need to worry about threading issues with indexes
365 private boolean[] frameIsBeat;
366
367 private final int startFrame;
368 private final int frameLength; // in frames
369 private final WaveFormRenderer renderer;
370 private final boolean recommendInvalidations;
371
372 public WaveFormRenderTask(
373 Image imageBuffer,
374 byte[] audioBytes,
375 AudioFormat format,
376 List<Double> detectedBeats,
377 int startFrame,
378 int frameLength,
379 boolean recommendInvalidations) {
380
381 assert(audioBytes != null);
382 assert(format != null);
383 assert(imageBuffer != null);
384 assert(((startFrame + frameLength) * format.getFrameSize()) <= audioBytes.length);
385
386 this.imageBuffer = imageBuffer;
387 this.audioBytes = audioBytes;
388 this.startFrame = startFrame;
389 this.frameLength = frameLength;
390 this.recommendInvalidations = recommendInvalidations;
391
392 this.frameIsBeat = new boolean[frameLength];
393 if (detectedBeats != null) {
394 for (double beat_in_secs: detectedBeats){
395 int frame_pos = (int)Math.round(beat_in_secs * format.getSampleRate());
396 frameIsBeat[frame_pos] = true;
397 }
398 }
399
400 renderer = new DualPeakTroughWaveFormRenderer(format);
401 }
402
403 private void setState(int newState) {
404 this.state = newState;
405
406 if (keepRendering) { // invalidate if cancel not requested
407 recommendInvalidation();
408 }
409
410 // Nullify expensive references when stopped so they can be freed by the garbage collector
411 if (state == STATE_STOPPED) {
412 audioBytes = null;
413 frameIsBeat = null;
414 imageBuffer = null;
415 }
416
417 }
418
419 /**
420 *
421 * @return
422 * True if this is rendering .. thus being proccessed.
423 */
424 public boolean isRendering() {
425 return state == STATE_RENDERING;
426 }
427
428
429 /**
430 * <b>WARNING:</b> Think about the race conditions ... now cannot no when this may start therefore
431 * it is always safe to lock buffer if in pending state.
432 *
433 * @return
434 * True if this has not started to render
435 */
436 public boolean hasStarted() {
437 return state != STATE_PENDING;
438 }
439
440 /**
441 *
442 * @return
443 * True if has stopped
444 */
445 public boolean isStopped() {
446 return state == STATE_STOPPED;
447
448 }
449
450 private void recommendInvalidation() {
451 if (recommendInvalidations)
452 fireSubjectChangedLaterOnSwingThread(
453 new SubjectChangedEvent(ApolloSubjectChangedEvent.RENDER_TASK_INVALIDATION_RECOMENDATION));
454 }
455
456 }
457
458}
Note: See TracBrowser for help on using the repository browser.