source: trunk/src/org/apollo/agents/MelodySearch.java@ 1102

Last change on this file since 1102 was 1102, checked in by davidb, 6 years ago

Reworking of the code-base to separate logic from graphics. This version of Expeditee now supports a JFX graphics as an alternative to SWING
File size: 13.0 KB
Line 
1package org.apollo.agents;
2
3import java.io.File;
4import java.io.FileNotFoundException;
5import java.io.IOException;
6import java.util.Collections;
7import java.util.LinkedList;
8import java.util.List;
9
10import javax.sound.sampled.AudioFormat;
11
12import org.apollo.ApolloSystem;
13import org.apollo.io.AudioPathManager;
14import org.apollo.meldex.DynamicProgrammingAlgorithm;
15import org.apollo.meldex.McNabMongeauSankoffAlgorithm;
16import org.apollo.meldex.MeldexConversion;
17import org.apollo.meldex.Melody;
18import org.apollo.meldex.StandardisedMelody;
19import org.apollo.meldex.WavSample;
20import org.apollo.util.ExpediteeFileTextSearch;
21import org.apollo.util.TextItemSearchResult;
22import org.apollo.widgets.LinkedTrack;
23import org.apollo.widgets.SampledTrack;
24import org.apollo.widgets.TrackWidgetCommons;
25import org.expeditee.agents.SearchAgent;
26import org.expeditee.core.Colour;
27import org.expeditee.gui.DisplayController;
28import org.expeditee.gui.Frame;
29import org.expeditee.gui.FrameIO;
30import org.expeditee.settings.folders.FolderSettings;
31import org.expeditee.items.Item;
32import org.expeditee.items.ItemUtils;
33import org.expeditee.items.widgets.Widget;
34import org.expeditee.items.widgets.WidgetCorner;
35import org.expeditee.items.widgets.WidgetEdge;
36
37/**
38 * Performs a melody search against track widgets within a whole frameset.
39 *
40 * Uses meldex. Thanks David Bainbridge.
41 *
42 * The agent runs a query on the given track widget that launched it.
43 * If the track launches it it does a full search for all tracks on the current frameset.
44 *
45 * @author Brook Novak
46 *
47 */
48public class MelodySearch extends SearchAgent {
49
50 private long firstFrame = 1;
51
52 private long maxFrame = Integer.MAX_VALUE;
53
54 /** Either querry from raw audio or track widget: */
55 private SampledTrack querryTrack = null;
56
57 /** Either querry from raw audio or track widget: */
58 private byte[] querryRawAudio = null;
59 private AudioFormat querryRawAudioFormat = null;
60
61 public static final String MELODY_METAFILE_PREFIX = ".";
62 public static final String MELODY_METAFILE_SUFFIX = ".mel";
63
64 /** The default score cut-off for omitting bad results above this value*/
65 private static final float DEFAULT_SCORE_THRESHOLD = 400.0f;
66
67 public MelodySearch() {
68 super("MelodySearch");
69 }
70
71 /**
72 *
73 * @param firstFrame
74 * The first frame number to start searching from (inclusive)
75 *
76 * @param maxFrame
77 * The max frame number to start searching from (inclusive)
78 */
79 public MelodySearch(long firstFrame, long maxFrame) {
80 super("MelodySearch");
81 this.firstFrame = firstFrame;
82 this.maxFrame = maxFrame;
83 }
84
85 public void useRawAudio(byte[] querryRawAudio, AudioFormat querryRawAudioFormat) {
86
87 if (querryRawAudio == null || querryRawAudioFormat == null) return;
88 this.querryRawAudio = querryRawAudio;
89 this.querryRawAudioFormat = querryRawAudioFormat;
90 }
91
92 /**
93 * {@inheritDoc}
94 */
95 @Override
96 public boolean initialise(Frame frame, Item item)
97 {
98 if (!super.initialise(frame, item)) return false;
99
100 // Get the track to querry .. if given one
101 querryTrack = null;
102
103 if (item != null) {
104
105 Widget iw = null;
106
107 if (item instanceof WidgetCorner) {
108
109 iw = ((WidgetCorner)item).getWidgetSource();
110 } if (item instanceof WidgetEdge) {
111
112 iw = ((WidgetEdge)item).getWidgetSource();
113 }
114
115 if (iw != null && iw instanceof SampledTrack) {
116 querryTrack = (SampledTrack)iw;
117 }
118
119 }
120
121 return true;
122 }
123
124 /**
125 * {@inheritDoc}
126 */
127 @Override
128 protected Frame process(Frame frame) {
129
130 try {
131 if(frame == null) {
132 frame = FrameIO.LoadFrame(_startName + '0');
133 }
134
135 String path = frame.getPath();
136
137 int count = FrameIO.getLastNumber(_startName);
138
139 String trackPrefix = ItemUtils.GetTag(ItemUtils.TAG_IWIDGET) + ": ";
140 String linkedTrackPrefix = trackPrefix;
141
142 trackPrefix += SampledTrack.class.getName();
143 linkedTrackPrefix += LinkedTrack.class.getName();
144
145 Melody querryMelody = null;
146 // Maps FrameName -> MelodySearchResult
147 List<MelodySearchResult> melodyScores = new LinkedList<MelodySearchResult>();
148
149 // If querrying a track widget then get its melody
150 if (querryTrack != null) {
151 assert(querryRawAudio == null);
152 assert(querryRawAudioFormat == null);
153
154 try {
155 querryMelody = MeldexConversion.toMelody(querryTrack);
156 } catch (IOException e) {
157 e.printStackTrace();
158 } catch (OutOfMemoryError ex) {
159 ex.printStackTrace();
160 }
161
162 if (querryMelody == null) { // abort - failed to get audio
163 overwriteMessage("Melody search aborted: Failed to load tracks audio");
164 _results.addText("Melody search aborted: querry data not good enough to search with",
165 Colour.RED, null, null, false);
166 _results.addText("Click here for help on melody searches",
167 Colour.FromRGB255(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
168
169 _results.save();
170 return null;
171 }
172
173 // If querrying raw audio then get its melody
174 } else if (querryRawAudio != null && querryRawAudioFormat != null) {
175 assert(querryTrack == null);
176
177 try {
178 querryMelody = MeldexConversion.toMelody(querryRawAudio, querryRawAudioFormat);
179 } catch (IOException e) {
180 e.printStackTrace();
181 } catch (OutOfMemoryError ex) {
182 ex.printStackTrace();
183 }
184
185 if (querryMelody == null) { // abort - failed to get audio
186 overwriteMessage("Melody search aborted: Failed to proccess querry data");
187 _results.addText("Melody search aborted: querry data not good enough to search with", Colour.RED, null, null, false);
188 _results.addText("Click here for help on melody searches",
189 Colour.FromRGB255(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
190 _results.save();
191 return null;
192 }
193
194
195 }
196
197 // Support range searching... i.e. frame100 - frame500
198 for (long i = firstFrame;i <= maxFrame && i <= count; i++) {
199
200 // Has requested stop?
201 if (_stop) {
202 break;
203 }
204
205 String frameName = _startName + i;
206
207
208 overwriteMessage("Searching " + frameName); // RISKY
209 // Note: cannot invoke later otherwise can congest the swing queue!
210
211 // Perform prefix search
212 List<TextItemSearchResult> results = null;
213 try {
214
215 String fullpath = getFullPath(frameName, path);
216
217 if (fullpath != null) {
218 results = ExpediteeFileTextSearch.prefixSearch(
219 fullpath,
220 new String[] {trackPrefix, linkedTrackPrefix});
221 }
222
223 } catch (FileNotFoundException e) {
224 e.printStackTrace();
225 } catch (IOException e) {
226 e.printStackTrace();
227 }
228
229 if (results == null) continue; // frame does not exist or an error occured
230
231 // If the frame exists / succeeded to perform a prefix search then increment the
232 // searched frame count...
233 _frameCount++;
234
235 // Is doing a full search for all tracks?
236 if (querryMelody == null) {
237 if(!results.isEmpty()) {
238 _results.addText(frameName + "(" + results.size() + ")", null, frameName, null, false);
239 DisplayController.requestRefresh(true);
240 }
241
242 } else { // meldex querry
243
244 MelodySearchResult bestScore = null;
245
246 for (TextItemSearchResult res : results) {
247 if (res.data == null) continue;
248
249 // should this widget be indexed?
250 if (res.containsData(SampledTrack.META_DONT_INDEX_AUDIO_TAG)) continue;
251
252 // get the local filename
253 String localFilename = null;
254 String trackName = null;
255
256 // Parse meta
257 for (String data : res.data) {
258 if (data.startsWith(SampledTrack.META_LOCALNAME_TAG) &&
259 data.length() > SampledTrack.META_LOCALNAME_TAG.length()) {
260 localFilename = data.substring(SampledTrack.META_LOCALNAME_TAG.length());
261 if (trackName != null) break;
262 } else if (data.startsWith(TrackWidgetCommons.META_NAME_TAG) &&
263 data.length() > TrackWidgetCommons.META_NAME_TAG.length()) {
264 trackName = data.substring(TrackWidgetCommons.META_NAME_TAG.length());
265 if (localFilename != null) break;
266 }
267
268 }
269
270 if (localFilename == null)
271 continue;
272
273 // Safety: omit this if it is infact the very widget we are searching
274 if (querryTrack != null && querryTrack.getLocalFileName().equals(localFilename))
275 continue;
276
277 Melody testMelody = null;
278
279 // Get cached melody from file if it is up to date
280 String metaFilePath =
281 AudioPathManager.AUDIO_HOME_DIRECTORY
282 + MELODY_METAFILE_PREFIX
283 + localFilename
284 + MELODY_METAFILE_SUFFIX;
285
286 File localFile = new File(AudioPathManager.AUDIO_HOME_DIRECTORY + localFilename);
287 if (!localFile.exists()) continue;
288
289 File metaFile = new File(metaFilePath);
290
291 // If there is a metafile that contains the serialized melody and is up to date...
292 if (metaFile.exists() && metaFile.lastModified() >= localFile.lastModified()) {
293
294 try {
295 StandardisedMelody sm = StandardisedMelody.readMelodyFromFile(metaFilePath);
296 if (sm != null && sm instanceof Melody) {
297 testMelody = (Melody)sm;
298 }
299 } catch (FileNotFoundException e) {
300 e.printStackTrace();
301 } catch (IOException e) {
302 e.printStackTrace();
303 } catch (ClassNotFoundException e) {
304 e.printStackTrace();
305 }
306
307 }
308
309 if (_stop) break;
310
311 // If did not manage to load any melody meta ... calculate the meta
312 if (testMelody == null) {
313
314 // Load wave file
315 WavSample sampleTrack = new WavSample();
316 if (!sampleTrack.loadFromFile(localFile)) {
317 continue;
318 }
319 if (_stop) break;
320
321 // Transcribe melody
322 try {
323 testMelody = MeldexConversion.toMelody(sampleTrack);
324 } catch (IOException ex) {
325 ex.printStackTrace();
326 } catch (OutOfMemoryError ex) {
327 ex.printStackTrace();
328 }
329
330 if (testMelody == null) continue;
331 if (_stop) break;
332
333 // Save meta
334 if (metaFile.exists()) metaFile.delete();
335
336 try {
337 StandardisedMelody.writeMelodyToFile(metaFilePath, testMelody);
338 } catch (FileNotFoundException e) {
339 e.printStackTrace();
340 } catch (IOException e) {
341 e.printStackTrace();
342 }
343
344 }
345
346 assert(testMelody != null);
347 assert(querryMelody != null);
348
349 // Omit bogus melody transcriptions for which audio must not of have enough rests
350 // to give a strong enough representation. This avoids bogus meta getting better results
351 // than something actually more meaningful.
352 if (testMelody.getLength() <= 1) continue;
353
354 DynamicProgrammingAlgorithm dpa = new McNabMongeauSankoffAlgorithm();
355
356 float score = dpa.matchToPattern(querryMelody, testMelody, DynamicProgrammingAlgorithm.MATCH_ANYWHERE);
357
358 if (bestScore == null || bestScore.getScore() < score) {
359
360 bestScore = new MelodySearchResult(
361 frameName,
362 score,
363 trackName,
364 localFilename
365 );
366 }
367
368
369 } // next result in frame
370
371 if (_stop) {
372 break;
373 }
374
375 if (bestScore != null) {
376 melodyScores.add(bestScore);
377 }
378
379 }
380
381
382 } // Next frame
383
384
385 // Did do melody matching? add ordered results
386 if (querryMelody != null) {
387
388 float threshold = DEFAULT_SCORE_THRESHOLD; // TODO: Have tolerance option
389
390 // Order results descending
391 Collections.sort(melodyScores);
392
393 if (melodyScores.isEmpty() || melodyScores.get(0).getScore() > threshold) {
394 _results.addText("No matches", Colour.RED, null, null, false);
395 _results.addText("Click here to find out how to improve your melody searches",
396 Colour.FromRGB255(0, 180, 0), ApolloSystem.HELP_MELODYSEARCH_FRAMENAME, null, false);
397 } else {
398
399 int rank = 1;
400 for (MelodySearchResult melRes : melodyScores) {
401
402 if (melRes.getScore() <= threshold) {
403
404 String name = melRes.getTrackName();
405 if (name == null || name.length() == 0)
406 name = "Unnamed";
407
408 _results.addText(rank + ": " + melRes.getParentFrame() + " ("
409 + name + ")",
410 null,
411 melRes.getParentFrame(), null, false);
412
413
414 }
415
416 rank ++;
417
418 }
419
420 }
421
422 }
423
424 if (_stop) {
425 _results.addText("Search cancelled", Colour.RED, null, null, false);
426 }
427
428 // Spit out result(s)
429 _results.save();
430
431 String resultFrameName = _results.getName();
432 if (_clicked != null)
433 _clicked.setLink(resultFrameName);
434
435 return _results.getFirstFrame();
436
437 }
438 finally {
439 querryRawAudio = null; // Free memory
440 }
441
442 }
443
444 /**
445 * Gets the full path from a given framename and path
446 * @param frameName
447 * @param path
448 * @return
449 */
450 private String getFullPath(String frameName, String path) {
451
452 String fullPath = null;
453 if (path == null) {
454 for (String possiblePath : FolderSettings.FrameDirs.get()) {
455 fullPath = FrameIO.getFrameFullPathName(possiblePath, frameName);
456 if (fullPath != null)
457 break;
458 }
459 } else {
460 fullPath = FrameIO.getFrameFullPathName(path, frameName);
461 }
462
463 return fullPath;
464
465 }
466}
Note: See TracBrowser for help on using the repository browser.