source: trunk/faces/apollo/src/org/apollo/agents/MelodySearch.java@ 903

Last change on this file since 903 was 903, checked in by davidb, 10 years ago

Changes needed after restructuring of Settings code/classes
File size: 13.6 KB
Line 
1package org.apollo.agents;
2
3import java.awt.Color;
4import java.io.File;
5import java.io.FileNotFoundException;
6import java.io.IOException;
7import java.util.Collections;
8import java.util.LinkedList;
9import java.util.List;
10
11import javax.sound.sampled.AudioFormat;
12import javax.swing.SwingUtilities;
13
14import org.apollo.ApolloSystem;
15import org.apollo.io.AudioPathManager;
16import org.apollo.meldex.DynamicProgrammingAlgorithm;
17import org.apollo.meldex.McNabMongeauSankoffAlgorithm;
18import org.apollo.meldex.MeldexConversion;
19import org.apollo.meldex.Melody;
20import org.apollo.meldex.StandardisedMelody;
21import org.apollo.meldex.WavSample;
22import org.apollo.util.ExpediteeFileTextSearch;
23import org.apollo.util.TextItemSearchResult;
24import org.apollo.widgets.LinkedTrack;
25import org.apollo.widgets.SampledTrack;
26import org.apollo.widgets.TrackWidgetCommons;
27import org.expeditee.agents.SearchAgent;
28import org.expeditee.gui.Frame;
29import org.expeditee.gui.FrameGraphics;
30import org.expeditee.gui.FrameIO;
31import org.expeditee.settings.UserSettings;
32import org.expeditee.settings.folders.FolderSettings;
33import org.expeditee.items.Item;
34import org.expeditee.items.ItemUtils;
35import org.expeditee.items.widgets.InteractiveWidget;
36import org.expeditee.items.widgets.WidgetCorner;
37import org.expeditee.items.widgets.WidgetEdge;
38
39/**
40 * Performs a melody search against track widgets within a whole frameset.
41 *
42 * Uses meldex. Thanks David Bainbridge.
43 *
44 * The agent runs a querry on the given track widget that launched it.
45 * If the track launches it it does a full search for all tracks on the current frameset.
46 *
47 * @author Brook Novak
48 *
49 */
50public class MelodySearch extends SearchAgent {
51
52 private long firstFrame = 1;
53
54 private long maxFrame = Integer.MAX_VALUE;
55
56 /** Either querry from raw audio or track widget: */
57 private SampledTrack querryTrack = null;
58
59 /** Either querry from raw audio or track widget: */
60 private byte[] querryRawAudio = null;
61 private AudioFormat querryRawAudioFormat = null;
62
63 public static final String MELODY_METAFILE_PREFIX = ".";
64 public static final String MELODY_METAFILE_SUFFIX = ".mel";
65
66 /** The default score cut-off for omitting bad results above this value*/
67 private static final float DEFAULT_SCORE_THRESHOLD = 400.0f;
68
69 public MelodySearch() {
70 super("MelodySearch");
71 }
72
73 /**
74 *
75 * @param firstFrame
76 * The first frame number to start searching from (inclusive)
77 *
78 * @param maxFrame
79 * The max frame number to start searching from (inclusive)
80 */
81 public MelodySearch(long firstFrame, long maxFrame) {
82 super("MelodySearch");
83 this.firstFrame = firstFrame;
84 this.maxFrame = maxFrame;
85 }
86
87 public void useRawAudio(byte[] querryRawAudio, AudioFormat querryRawAudioFormat) {
88
89 if (querryRawAudio == null || querryRawAudioFormat == null) return;
90 this.querryRawAudio = querryRawAudio;
91 this.querryRawAudioFormat = querryRawAudioFormat;
92 }
93
94 /**
95 * {@inheritDoc}
96 */
97 @Override
98 public boolean initialise(Frame frame, Item item) {
99 if (!super.initialise(frame, item)) return false;
100
101 // Get the track to querry .. if given one
102 querryTrack = null;
103
104 if (item != null) {
105
106 InteractiveWidget iw = null;
107
108 if (item instanceof WidgetCorner) {
109
110 iw = ((WidgetCorner)item).getWidgetSource();
111 } if (item instanceof WidgetEdge) {
112
113 iw = ((WidgetEdge)item).getWidgetSource();
114 }
115
116 if (iw != null && iw instanceof SampledTrack) {
117 querryTrack = (SampledTrack)iw;
118 }
119
120 }
121
122 return true;
123 }
124
125 /**
126 * {@inheritDoc}
127 */
128 @Override
129 protected Frame process(Frame frame) {
130
131 try {
132 if(frame == null) {
133 frame = FrameIO.LoadFrame(_startName + '0');
134 }
135
136 String path = frame.getPath();
137
138 int count = FrameIO.getLastNumber(_startName);
139
140 String trackPrefix = ItemUtils.GetTag(ItemUtils.TAG_IWIDGET) + ": ";
141 String linkedTrackPrefix = trackPrefix;
142
143 trackPrefix += SampledTrack.class.getName();
144 linkedTrackPrefix += LinkedTrack.class.getName();
145
146 Melody querryMelody = null;
147 // Maps FrameName -> MelodySearchResult
148 List<MelodySearchResult> melodyScores = new LinkedList<MelodySearchResult>();
149
150 // If querrying a track widget then get its melody
151 if (querryTrack != null) {
152 assert(querryRawAudio == null);
153 assert(querryRawAudioFormat == null);
154
155 try {
156 querryMelody = MeldexConversion.toMelody(querryTrack);
157 } catch (IOException e) {
158 e.printStackTrace();
159 } catch (OutOfMemoryError ex) {
160 ex.printStackTrace();
161 }
162
163 if (querryMelody == null) { // abort - failed to get audio
164 SwingUtilities.invokeLater(new ExpediteeMessageBayFeedback(
165 "Melody search aborted: Failed to load tracks audio"));
166 _results.addText("Melody search aborted: querry data not good enough to search with",
167 Color.RED, null, null, false);
168 _results.addText("Click here for help on melody searches",
169 new Color(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
170
171 _results.save();
172 return null;
173 }
174
175 // If querrying raw audio then get its melody
176 } else if (querryRawAudio != null && querryRawAudioFormat != null) {
177 assert(querryTrack == null);
178
179 try {
180 querryMelody = MeldexConversion.toMelody(querryRawAudio, querryRawAudioFormat);
181 } catch (IOException e) {
182 e.printStackTrace();
183 } catch (OutOfMemoryError ex) {
184 ex.printStackTrace();
185 }
186
187 if (querryMelody == null) { // abort - failed to get audio
188 SwingUtilities.invokeLater(new ExpediteeMessageBayFeedback(
189 "Melody search aborted: Failed to proccess querry data"));
190 _results.addText("Melody search aborted: querry data not good enough to search with", Color.RED, null, null, false);
191 _results.addText("Click here for help on melody searches",
192 new Color(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
193 _results.save();
194 return null;
195 }
196
197
198 }
199
200 // Support range searching... i.e. frame100 - frame500
201 for (long i = firstFrame;i <= maxFrame && i <= count; i++) {
202
203 // Has requested stop?
204 if (_stop) {
205 break;
206 }
207
208 String frameName = _startName + i;
209
210
211 overwriteMessage("Searching " + frameName); // RISKY
212 // Note: cannot invoke later otherwise can congest the swing queue!
213
214 // Perform prefix search
215 List<TextItemSearchResult> results = null;
216 try {
217
218 String fullpath = getFullPath(frameName, path);
219
220 if (fullpath != null) {
221 results = ExpediteeFileTextSearch.prefixSearch(
222 fullpath,
223 new String[] {trackPrefix, linkedTrackPrefix});
224 }
225
226 } catch (FileNotFoundException e) {
227 e.printStackTrace();
228 } catch (IOException e) {
229 e.printStackTrace();
230 }
231
232 if (results == null) continue; // frame does not exist or an error occured
233
234 // If the frame exists / succeeded to perform a prefix search then increment the
235 // searched frame count...
236 _frameCount++;
237
238 // Is doing a full search for all tracks?
239 if (querryMelody == null) {
240 if(!results.isEmpty()) {
241 _results.addText(frameName + "(" + results.size() + ")", null, frameName, null, false);
242 FrameGraphics.requestRefresh(true);
243 }
244
245 } else { // meldex querry
246
247 MelodySearchResult bestScore = null;
248
249 for (TextItemSearchResult res : results) {
250 if (res.data == null) continue;
251
252 // should this widget be indexed?
253 if (res.containsData(SampledTrack.META_DONT_INDEX_AUDIO_TAG)) continue;
254
255 // get the local filename
256 String localFilename = null;
257 String trackName = null;
258
259 // Parse meta
260 for (String data : res.data) {
261 if (data.startsWith(SampledTrack.META_LOCALNAME_TAG) &&
262 data.length() > SampledTrack.META_LOCALNAME_TAG.length()) {
263 localFilename = data.substring(SampledTrack.META_LOCALNAME_TAG.length());
264 if (trackName != null) break;
265 } else if (data.startsWith(TrackWidgetCommons.META_NAME_TAG) &&
266 data.length() > TrackWidgetCommons.META_NAME_TAG.length()) {
267 trackName = data.substring(TrackWidgetCommons.META_NAME_TAG.length());
268 if (localFilename != null) break;
269 }
270
271 }
272
273 if (localFilename == null)
274 continue;
275
276 // Safety: omit this if it is infact the very widget we are searching
277 if (querryTrack != null && querryTrack.getLocalFileName().equals(localFilename))
278 continue;
279
280 Melody testMelody = null;
281
282 // Get cached melody from file if it is up to date
283 String metaFilePath =
284 AudioPathManager.AUDIO_HOME_DIRECTORY
285 + MELODY_METAFILE_PREFIX
286 + localFilename
287 + MELODY_METAFILE_SUFFIX;
288
289 File localFile = new File(AudioPathManager.AUDIO_HOME_DIRECTORY + localFilename);
290 if (!localFile.exists()) continue;
291
292 File metaFile = new File(metaFilePath);
293
294 // If there is a metafile that contains the serialized melody and is up to date...
295 if (metaFile.exists() && metaFile.lastModified() >= localFile.lastModified()) {
296
297 try {
298 StandardisedMelody sm = StandardisedMelody.readMelodyFromFile(metaFilePath);
299 if (sm != null && sm instanceof Melody) {
300 testMelody = (Melody)sm;
301 }
302 } catch (FileNotFoundException e) {
303 e.printStackTrace();
304 } catch (IOException e) {
305 e.printStackTrace();
306 } catch (ClassNotFoundException e) {
307 e.printStackTrace();
308 }
309
310 }
311
312 if (_stop) break;
313
314 // If did not manage to load any melody meta ... calculate the meta
315 if (testMelody == null) {
316
317 // Load wave file
318 WavSample sampleTrack = new WavSample();
319 if (!sampleTrack.loadFromFile(localFile)) {
320 continue;
321 }
322 if (_stop) break;
323
324 // Transcribe melody
325 try {
326 testMelody = MeldexConversion.toMelody(sampleTrack);
327 } catch (IOException ex) {
328 ex.printStackTrace();
329 } catch (OutOfMemoryError ex) {
330 ex.printStackTrace();
331 }
332
333 if (testMelody == null) continue;
334 if (_stop) break;
335
336 // Save meta
337 if (metaFile.exists()) metaFile.delete();
338
339 try {
340 StandardisedMelody.writeMelodyToFile(metaFilePath, testMelody);
341 } catch (FileNotFoundException e) {
342 e.printStackTrace();
343 } catch (IOException e) {
344 e.printStackTrace();
345 }
346
347 }
348
349 assert(testMelody != null);
350 assert(querryMelody != null);
351
352 // Omit bogus melody transcriptions for which audio must not of have enough rests
353 // to give a strong enough representation. This avoids bogus meta getting better results
354 // than something actually more meaningful.
355 if (testMelody.getLength() <= 1) continue;
356
357 DynamicProgrammingAlgorithm dpa = new McNabMongeauSankoffAlgorithm();
358
359 float score = dpa.matchToPattern(querryMelody, testMelody, DynamicProgrammingAlgorithm.MATCH_ANYWHERE);
360
361 if (bestScore == null || bestScore.getScore() < score) {
362
363 bestScore = new MelodySearchResult(
364 frameName,
365 score,
366 trackName,
367 localFilename
368 );
369 }
370
371
372 } // next result in frame
373
374 if (_stop) {
375 break;
376 }
377
378 if (bestScore != null) {
379 melodyScores.add(bestScore);
380 }
381
382 }
383
384
385 } // Next frame
386
387
388 // Did do melody matching? add ordered results
389 if (querryMelody != null) {
390
391 float threshold = DEFAULT_SCORE_THRESHOLD; // TODO: Have tolerance option
392
393 // Order results descending
394 Collections.sort(melodyScores);
395
396 if (melodyScores.isEmpty() || melodyScores.get(0).getScore() > threshold) {
397 _results.addText("No matches", Color.RED, null, null, false);
398 _results.addText("Click here to find out how to improve your melody searches",
399 new Color(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
400 } else {
401
402 int rank = 1;
403 for (MelodySearchResult melRes : melodyScores) {
404
405 if (melRes.getScore() <= threshold) {
406
407 String name = melRes.getTrackName();
408 if (name == null || name.length() == 0)
409 name = "Unnamed";
410
411 _results.addText(rank + ": " + melRes.getParentFrame() + " ("
412 + name + ")",
413 null,
414 melRes.getParentFrame(), null, false);
415
416
417 }
418
419 rank ++;
420
421 }
422
423 }
424
425 }
426
427 if (_stop) {
428 _results.addText("Search cancelled", Color.RED, null, null, false);
429 }
430
431 // Spit out result(s)
432 _results.save();
433
434 String resultFrameName = _results.getName();
435 if (_clicked != null)
436 _clicked.setLink(resultFrameName);
437
438 return _results.getFirstFrame();
439
440 }
441 finally {
442 querryRawAudio = null; // Free memory
443 }
444
445 }
446
447
448
449
450 /**
451 * Gets the full path from a given framename and path
452 * @param frameName
453 * @param path
454 * @return
455 */
456 private String getFullPath(String frameName, String path) {
457
458 String fullPath = null;
459 if (path == null) {
460 for (String possiblePath : FolderSettings.FrameDirs.get()) {
461 fullPath = FrameIO.getFrameFullPathName(possiblePath, frameName);
462 if (fullPath != null)
463 break;
464 }
465 } else {
466 fullPath = FrameIO.getFrameFullPathName(path, frameName);
467 }
468
469 return fullPath;
470
471 }
472
473 /**
474 * Safely outputs a message on the messagebay .. if ran on swing thread.
475 * @author Brook Novak
476 *
477 */
478 private class ExpediteeMessageBayFeedback implements Runnable {
479
480 String feedbackMessage;
481
482 public ExpediteeMessageBayFeedback(String feedbackMessage) {
483 assert(feedbackMessage != null);
484 this.feedbackMessage = feedbackMessage;
485 }
486
487
488 public void run() {
489 assert(feedbackMessage != null);
490 overwriteMessage(feedbackMessage);
491 }
492 }
493
494
495}
Note: See TracBrowser for help on using the repository browser.