source: trunk/src_apollo/org/apollo/agents/MelodySearch.java@ 318

Last change on this file since 318 was 318, checked in by bjn8, 16 years ago

Refactored a class name and extended recorder widgets to have a perminant lifetime option (for optimum idea capturing!)
File size: 13.6 KB
Line 
1package org.apollo.agents;
2
3import java.awt.Color;
4import java.io.File;
5import java.io.FileNotFoundException;
6import java.io.IOException;
7import java.util.Collections;
8import java.util.LinkedList;
9import java.util.List;
10
11import javax.sound.sampled.AudioFormat;
12import javax.swing.SwingUtilities;
13
14import org.apollo.ApolloSystem;
15import org.apollo.io.AudioPathManager;
16import org.apollo.meldex.DynamicProgrammingAlgorithm;
17import org.apollo.meldex.McNabMongeauSankoffAlgorithm;
18import org.apollo.meldex.MeldexConversion;
19import org.apollo.meldex.Melody;
20import org.apollo.meldex.StandardisedMelody;
21import org.apollo.meldex.WavSample;
22import org.apollo.util.ExpediteeFileTextSearch;
23import org.apollo.util.TextItemSearchResult;
24import org.apollo.widgets.LinkedTrack;
25import org.apollo.widgets.SampledTrack;
26import org.apollo.widgets.TrackWidgetCommons;
27import org.expeditee.agents.SearchAgent;
28import org.expeditee.gui.Frame;
29import org.expeditee.gui.FrameGraphics;
30import org.expeditee.gui.FrameIO;
31import org.expeditee.gui.UserSettings;
32import org.expeditee.items.Item;
33import org.expeditee.items.ItemUtils;
34import org.expeditee.items.widgets.InteractiveWidget;
35import org.expeditee.items.widgets.WidgetCorner;
36import org.expeditee.items.widgets.WidgetEdge;
37
38/**
39 * Performs a melody search against track widgets within a whole frameset.
40 *
41 * Uses meldex. Thanks David Bainbridge.
42 *
43 * The agent runs a querry on the given track widget that launched it.
44 * If the track launches it it does a full search for all tracks on the current frameset.
45 *
46 * @author Brook Novak
47 *
48 */
49public class MelodySearch extends SearchAgent {
50
51 private long firstFrame = 1;
52
53 private long maxFrame = Integer.MAX_VALUE;
54
55 /** Either querry from raw audio or track widget: */
56 private SampledTrack querryTrack = null;
57
58 /** Either querry from raw audio or track widget: */
59 private byte[] querryRawAudio = null;
60 private AudioFormat querryRawAudioFormat = null;
61
62 private static final String MELODY_METAFILE_PREFIX = ".";
63 private static final String MELODY_METAFILE_SUFFIX = ".mel";
64
65 /** The default score cut-off for omitting bad results above this value*/
66 private static final float DEFAULT_SCORE_THRESHOLD = 400.0f;
67
68 public MelodySearch() {
69 super("MelodySearch");
70 }
71
72 /**
73 *
74 * @param firstFrame
75 * The first frame number to start searching from (inclusive)
76 *
77 * @param maxFrame
78 * The max frame number to start searching from (inclusive)
79 */
80 public MelodySearch(long firstFrame, long maxFrame) {
81 super("MelodySearch");
82 this.firstFrame = firstFrame;
83 this.maxFrame = maxFrame;
84 }
85
86 public void useRawAudio(byte[] querryRawAudio, AudioFormat querryRawAudioFormat) {
87
88 if (querryRawAudio == null || querryRawAudioFormat == null) return;
89 this.querryRawAudio = querryRawAudio;
90 this.querryRawAudioFormat = querryRawAudioFormat;
91 }
92
93 /**
94 * {@inheritDoc}
95 */
96 @Override
97 public boolean initialise(Frame frame, Item item) {
98 if (!super.initialise(frame, item)) return false;
99
100 // Get the track to querry .. if given one
101 querryTrack = null;
102
103 if (item != null) {
104
105 InteractiveWidget iw = null;
106
107 if (item instanceof WidgetCorner) {
108
109 iw = ((WidgetCorner)item).getWidgetSource();
110 } if (item instanceof WidgetEdge) {
111
112 iw = ((WidgetEdge)item).getWidgetSource();
113 }
114
115 if (iw != null && iw instanceof SampledTrack) {
116 querryTrack = (SampledTrack)iw;
117 }
118
119 }
120
121 return true;
122 }
123
124 /**
125 * {@inheritDoc}
126 */
127 @Override
128 protected Frame process(Frame frame) {
129
130 try {
131 if(frame == null) {
132 frame = FrameIO.LoadFrame(_startName + '0');
133 }
134
135 String path = frame.getPath();
136
137 int count = FrameIO.getLastNumber(_startName);
138
139 String trackPrefix = ItemUtils.GetTag(ItemUtils.TAG_IWIDGET) + ": ";
140 String linkedTrackPrefix = trackPrefix;
141
142 trackPrefix += SampledTrack.class.getName();
143 linkedTrackPrefix += LinkedTrack.class.getName();
144
145 Melody querryMelody = null;
146 // Maps FrameName -> MelodySearchResult
147 List<MelodySearchResult> melodyScores = new LinkedList<MelodySearchResult>();
148
149 // If querrying a track widget then get its melody
150 if (querryTrack != null) {
151 assert(querryRawAudio == null);
152 assert(querryRawAudioFormat == null);
153
154 try {
155 querryMelody = MeldexConversion.toMelody(querryTrack);
156 } catch (IOException e) {
157 e.printStackTrace();
158 } catch (OutOfMemoryError ex) {
159 ex.printStackTrace();
160 }
161
162 if (querryMelody == null) { // abort - failed to get audio
163 SwingUtilities.invokeLater(new ExpediteeMessageBayFeedback(
164 "Melody search aborted: Failed to load tracks audio"));
165 _results.addText("Melody search aborted: querry data not good enough to search with",
166 Color.RED, null, null, false);
167 _results.addText("Click here for help on melody searches",
168 new Color(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
169
170 _results.save();
171 return null;
172 }
173
174 // If querrying raw audio then get its melody
175 } else if (querryRawAudio != null && querryRawAudioFormat != null) {
176 assert(querryTrack == null);
177
178 try {
179 querryMelody = MeldexConversion.toMelody(querryRawAudio, querryRawAudioFormat);
180 } catch (IOException e) {
181 e.printStackTrace();
182 } catch (OutOfMemoryError ex) {
183 ex.printStackTrace();
184 }
185
186 if (querryMelody == null) { // abort - failed to get audio
187 SwingUtilities.invokeLater(new ExpediteeMessageBayFeedback(
188 "Melody search aborted: Failed to proccess querry data"));
189 _results.addText("Melody search aborted: querry data not good enough to search with", Color.RED, null, null, false);
190 _results.addText("Click here for help on melody searches",
191 new Color(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
192 _results.save();
193 return null;
194 }
195
196
197 }
198
199 // Support range searching... i.e. frame100 - frame500
200 for (long i = firstFrame;i <= maxFrame && i <= count; i++) {
201
202 // Has requested stop?
203 if (_stop) {
204 break;
205 }
206
207 String frameName = _startName + i;
208
209
210 overwriteMessage("Searching " + frameName); // RISKY
211 // Note: cannot invoke later otherwise can congest the swing queue!
212
213 // Perform prefix search
214 List<TextItemSearchResult> results = null;
215 try {
216
217 String fullpath = getFullPath(frameName, path);
218
219 if (fullpath != null) {
220 results = ExpediteeFileTextSearch.prefixSearch(
221 fullpath,
222 new String[] {trackPrefix, linkedTrackPrefix});
223 }
224
225 } catch (FileNotFoundException e) {
226 e.printStackTrace();
227 } catch (IOException e) {
228 e.printStackTrace();
229 }
230
231 if (results == null) continue; // frame does not exist or an error occured
232
233 // If the frame exists / succeeded to perform a prefix search then increment the
234 // searched frame count...
235 _frameCount++;
236
237 // Is doing a full search for all tracks?
238 if (querryMelody == null) {
239 if(!results.isEmpty()) {
240 _results.addText(frameName + "(" + results.size() + ")", null, frameName, null, false);
241 FrameGraphics.requestRefresh(true);
242 }
243
244 } else { // meldex querry
245
246 MelodySearchResult bestScore = null;
247
248 for (TextItemSearchResult res : results) {
249 if (res.data == null) continue;
250
251 // should this widget be indexed?
252 if (res.containsData(SampledTrack.META_DONT_INDEX_AUDIO_TAG)) continue;
253
254 // get the local filename
255 String localFilename = null;
256 String trackName = null;
257
258 // Parse meta
259 for (String data : res.data) {
260 if (data.startsWith(SampledTrack.META_LOCALNAME_TAG) &&
261 data.length() > SampledTrack.META_LOCALNAME_TAG.length()) {
262 localFilename = data.substring(SampledTrack.META_LOCALNAME_TAG.length());
263 if (trackName != null) break;
264 } else if (data.startsWith(TrackWidgetCommons.META_NAME_TAG) &&
265 data.length() > TrackWidgetCommons.META_NAME_TAG.length()) {
266 trackName = data.substring(TrackWidgetCommons.META_NAME_TAG.length());
267 if (localFilename != null) break;
268 }
269
270 }
271
272 if (localFilename == null)
273 continue;
274
275 // Safety: omit this if it is infact the very widget we are searching
276 if (querryTrack != null && querryTrack.getLocalFileName().equals(localFilename))
277 continue;
278
279 Melody testMelody = null;
280
281 // Get cached melody from file if it is up to date
282 String metaFilePath =
283 AudioPathManager.AUDIO_HOME_DIRECTORY
284 + MELODY_METAFILE_PREFIX
285 + localFilename
286 + MELODY_METAFILE_SUFFIX;
287
288 File localFile = new File(AudioPathManager.AUDIO_HOME_DIRECTORY + localFilename);
289 if (!localFile.exists()) continue;
290
291 File metaFile = new File(metaFilePath);
292
293 // If there is a metafile that contains the serialized melody and is up to date...
294 if (metaFile.exists() && metaFile.lastModified() >= localFile.lastModified()) {
295
296 try {
297 StandardisedMelody sm = StandardisedMelody.readMelodyFromFile(metaFilePath);
298 if (sm != null && sm instanceof Melody) {
299 testMelody = (Melody)sm;
300 }
301 } catch (FileNotFoundException e) {
302 e.printStackTrace();
303 } catch (IOException e) {
304 e.printStackTrace();
305 } catch (ClassNotFoundException e) {
306 e.printStackTrace();
307 }
308
309 }
310
311 if (_stop) break;
312
313 // If did not manage to load any melody meta ... calculate the meta
314 if (testMelody == null) {
315
316 // Load wave file
317 WavSample sampleTrack = new WavSample();
318 if (!sampleTrack.loadFromFile(localFile)) {
319 continue;
320 }
321 if (_stop) break;
322
323 // Transcribe melody
324 try {
325 testMelody = MeldexConversion.toMelody(sampleTrack);
326 } catch (IOException ex) {
327 ex.printStackTrace();
328 } catch (OutOfMemoryError ex) {
329 ex.printStackTrace();
330 }
331
332 if (testMelody == null) continue;
333 if (_stop) break;
334
335 // Save meta
336 if (metaFile.exists()) metaFile.delete();
337
338 try {
339 StandardisedMelody.writeMelodyToFile(metaFilePath, testMelody);
340 } catch (FileNotFoundException e) {
341 e.printStackTrace();
342 } catch (IOException e) {
343 e.printStackTrace();
344 }
345
346 }
347
348 assert(testMelody != null);
349 assert(querryMelody != null);
350
351 // Omit bogus melody transcriptions for which audio must not of have enough rests
352 // to give a strong enough representation. This avoids bogus meta getting better results
353 // than something actually more meaningful.
354 if (testMelody.getLength() <= 1) continue;
355
356 DynamicProgrammingAlgorithm dpa = new McNabMongeauSankoffAlgorithm();
357
358 float score = dpa.matchToPattern(querryMelody, testMelody, DynamicProgrammingAlgorithm.MATCH_ANYWHERE);
359
360 if (bestScore == null || bestScore.getScore() < score) {
361
362 bestScore = new MelodySearchResult(
363 frameName,
364 score,
365 trackName,
366 localFilename
367 );
368 }
369
370
371 } // next result in frame
372
373 if (_stop) {
374 break;
375 }
376
377 if (bestScore != null) {
378 melodyScores.add(bestScore);
379 }
380
381 }
382
383
384 } // Next frame
385
386
387 // Did do melody matching? add ordered results
388 if (querryMelody != null) {
389
390 float threshold = DEFAULT_SCORE_THRESHOLD; // TODO: Have tolerance option
391
392 // Order results descending
393 Collections.sort(melodyScores);
394
395 if (melodyScores.isEmpty() || melodyScores.get(0).getScore() > threshold) {
396 _results.addText("No matches", Color.RED, null, null, false);
397 _results.addText("Click here to find out how to improve your melody searches",
398 new Color(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
399 } else {
400
401 int rank = 1;
402 for (MelodySearchResult melRes : melodyScores) {
403
404 if (melRes.getScore() <= threshold) {
405
406 String name = melRes.getTrackName();
407 if (name == null || name.length() == 0)
408 name = "Unnamed";
409
410 _results.addText(rank + ": " + melRes.getParentFrame() + " ("
411 + name + ")",
412 null,
413 melRes.getParentFrame(), null, false);
414
415
416 }
417
418 rank ++;
419
420 }
421
422 }
423
424 }
425
426 if (_stop) {
427 _results.addText("Search cancelled", Color.RED, null, null, false);
428 }
429
430 // Spit out result(s)
431 _results.save();
432
433 String resultFrameName = _results.getName();
434 if (_clicked != null)
435 _clicked.setLink(resultFrameName);
436
437 return _results.getFirstFrame();
438
439 }
440 finally {
441 querryRawAudio = null; // Free memory
442 }
443
444 }
445
446
447
448
449 /**
450 * Gets the full path from a given framename and path
451 * @param frameName
452 * @param path
453 * @return
454 */
455 private String getFullPath(String frameName, String path) {
456
457 String fullPath = null;
458 if (path == null) {
459 for (String possiblePath : UserSettings.FrameDirs) {
460 fullPath = FrameIO.getFrameFullPathName(possiblePath, frameName);
461 if (fullPath != null)
462 break;
463 }
464 } else {
465 fullPath = FrameIO.getFrameFullPathName(path, frameName);
466 }
467
468 return fullPath;
469
470 }
471
472 /**
473 * Safely outputs a message on the messagebay .. if ran on swing thread.
474 * @author Brook Novak
475 *
476 */
477 private class ExpediteeMessageBayFeedback implements Runnable {
478
479 String feedbackMessage;
480
481 public ExpediteeMessageBayFeedback(String feedbackMessage) {
482 assert(feedbackMessage != null);
483 this.feedbackMessage = feedbackMessage;
484 }
485
486
487 public void run() {
488 assert(feedbackMessage != null);
489 overwriteMessage(feedbackMessage);
490 }
491 }
492
493
494}
Note: See TracBrowser for help on using the repository browser.