115 [[nodiscard]] std::span<const Event>
attacks() const noexcept
117 return isValid() ? std::span<const Event>(attacks_.get(), attackCount_)
118 : std::span<const Event>();
120 [[nodiscard]] std::span<const Event>
pulses() const noexcept
122 return isValid() ? std::span<const Event>(pulses_.get(), pulseCount_)
123 : std::span<const Event>();
125 [[nodiscard]] std::span<const FeatureFrame>
features() const noexcept
128 ? std::span<const FeatureFrame>(features_.get(), featureCount_)
129 : std::span<const FeatureFrame>();
142 return index < featureCount_
143 ?
static_cast<std::int64_t
>(index + 1) * hop_ - frameSize_ / 2
162 int frameSize_ = 0, hop_ = 0;
163 std::size_t attackCount_ = 0, pulseCount_ = 0, featureCount_ = 0;
164 std::unique_ptr<Event[]> attacks_, pulses_;
165 std::unique_ptr<FeatureFrame[]> features_;
197 detail::OfflineSession job(jobOptions);
199 candidate.options_ = options;
200 const auto spec = source.
getSpec();
202 if ((spec.sampleRate < 8000) || (spec.sampleRate > 384000))
205 const bool spectrumNeeded = options.attacks || options.pulses || options.retainFeatures;
207 candidate.hop_ = std::max(1,
static_cast<int>(std::lround(spec.sampleRate / 200)));
208 const int microHop = std::max(1,
static_cast<int>(std::lround(spec.sampleRate / 1000)));
209 const auto microCount =
static_cast<std::uint64_t
>((spec.frames - 1) / microHop + 1);
210 if (spec.frames > std::numeric_limits<std::int64_t>::max() - candidate.frameSize_)
212 const auto featureCount =
213 static_cast<std::uint64_t
>((spec.frames + candidate.frameSize_) / candidate.hop_);
224 static_cast<std::uint64_t
>(candidate.frameSize_ / 2 + 1),
sizeof(double)));
232 auto energy = OfflineEnergyAnalyzer<T>().analyze(
234 if (!energy.succeeded())
236 job.charge(energy.memoryBytes);
237 if (energy.analysis.getSpec() != spec)
239 candidate.energy_ = std::move(energy.analysis);
240 if (spectrumNeeded && candidate.energy_.samplePeak() > 0)
242 auto micro = std::make_unique<MicroBin[]>(
static_cast<std::size_t
>(microCount));
244 std::make_unique<FeatureFrame[]>(
static_cast<std::size_t
>(featureCount));
245 auto highPower = options.attacks
246 ? std::make_unique<double[]>(
static_cast<std::size_t
>(microCount))
248 auto highEvidence = options.attacks
249 ? std::make_unique<bool[]>(
static_cast<std::size_t
>(featureCount))
251 candidate.featureCount_ =
static_cast<std::size_t
>(featureCount);
252 scanFeatures(source, spec, candidate, microHop, micro.get(), highPower.get(),
253 highEvidence.get(), features.get(), job, jobOptions.
blockFrames);
254 const auto microView =
255 std::span<const MicroBin>(micro.get(),
static_cast<std::size_t
>(microCount));
256 const auto featureView =
257 std::span<const FeatureFrame>(features.get(), candidate.featureCount_);
258 const auto highView = std::span<const double>(
259 highPower.get(), highPower ?
static_cast<std::size_t
>(microCount) : 0);
260 const auto evidenceView = std::span<const bool>(
261 highEvidence.get(), highEvidence ? candidate.featureCount_ : 0);
263 buildEvents(candidate, featureView, microView, highView, evidenceView,
264 microHop,
false, job);
266 buildEvents(candidate, featureView, microView, {}, {}, microHop,
true, job);
267 if (options.retainFeatures)
268 candidate.features_ = std::move(features);
273 result.memoryBytes = job.bytes();
274 result.analysis = std::move(candidate);
288 double peakPower = 0;
289 std::int64_t peakFrame = 0;
296 std::span<const MicroBin> source;
297 std::span<const double> high;
298 PowerView(std::span<const MicroBin> full, std::span<const double> band = {})
noexcept
299 : source(full), high(band)
302 [[nodiscard]] std::size_t size() const noexcept {
return source.size(); }
303 [[nodiscard]]
double operator[](std::size_t i)
const noexcept
305 return high.empty() ? source[i].power : high[i];
309 static void scanFeatures(OfflineAudioSource<T> &source,
const OfflineAudioSpec &spec,
310 const Analysis &analysis,
int microHop, MicroBin *micro,
311 double *highPower,
bool *highEvidence,
312 FeatureFrame *features, detail::OfflineSession &job,
int blockFrames)
314 const int size = analysis.frameSize_, hop = analysis.hop_;
315 auto ring = std::make_unique<T[]>(
static_cast<std::size_t
>(2 * size * spec.channels));
316 detail::OnsetSpectrum<T> spectrum;
317 detail::OnsetNovelty<T> novelty;
318 spectrum.prepare(size);
319 novelty.prepare(spec.sampleRate, size,
true);
320 detail::OfflineBlock<T> scratch(job, spec, blockFrames);
321 Biquad<double> highPass;
322 std::unique_ptr<double[]> highGains;
326 highPass.setCoeffsNow(coefficients);
327 highGains = std::make_unique<double[]>(
static_cast<std::size_t
>(size / 2 + 1));
328 for (
int k = 0; k <= size / 2; ++k)
330 const double gain = coefficients.getMagnitude(
331 static_cast<double>(k) * spec.sampleRate / size, spec.sampleRate);
332 highGains[
static_cast<std::size_t
>(k)] = gain * gain;
335 OfflineFingerprint fingerprint;
336 int write = 0, inHop = 0;
337 std::size_t frame = 0;
338 auto push = [&](
const std::array<T, 2> &values) {
339 std::array<const T *, 2> pointers{};
340 for (
int c = 0; c < spec.channels; ++c)
342 T *channel = ring.get() +
static_cast<std::size_t
>(c * 2 * size);
343 channel[write] = channel[write + size] = values[
static_cast<std::size_t
>(c)];
350 for (
int c = 0; c < spec.channels; ++c)
351 pointers[
static_cast<std::size_t
>(c)] = ring.get() + c * 2 * size + write;
352 if (!spectrum.computePooled({pointers.data(), spec.channels, size}))
354 const auto magnitudes = spectrum.magnitudes();
355 double windowPower = 0;
357 double filteredLow = 0, filteredHigh = 0;
358 for (std::size_t k = 0; k < magnitudes.size(); ++k)
360 const double magnitude =
static_cast<double>(magnitudes[k]) / size;
362 magnitude * magnitude * (k == 0 || k + 1 == magnitudes.size() ? 1 : 2);
363 windowPower += power;
364 if (
static_cast<double>(k) * spec.sampleRate / size < 800)
368 filteredLow += power * highGains[k];
371 filteredHigh += power * highGains[k];
373 const auto value = novelty.process(spectrum.magnitudes(), {},
375 if (!value.valid || !std::isfinite(value.value) || !std::isfinite(value.spectralFlux))
381 highEvidence[frame] = filteredHigh > 0 && filteredHigh >= filteredLow;
382 features[frame++] = {value.value, value.spectralFlux, value.registers, windowPower,
383 windowPower > 0 ? lowPower / windowPower : 0};
385 const double normalization = analysis.energy_.samplePeak();
386 for (std::int64_t first = 0; first < spec.frames;)
389 static_cast<int>(std::min<std::int64_t>(scratch.frames(), spec.frames - first));
390 auto block = scratch.view(count);
392 for (
int f = 0; f < count; ++f)
394 std::array<T, 2> values{};
397 for (
int c = 0; c < spec.channels; ++c)
399 const T raw = block.getChannel(c)[f];
400 if (!std::isfinite(raw))
403 const double value =
static_cast<double>(raw) / normalization;
404 if (!std::isfinite(value) || std::abs(value) > 1)
406 values[
static_cast<std::size_t
>(c)] =
static_cast<T
>(value);
407 power += value * value;
410 const double filtered = highPass.processSample(value, c);
411 high += filtered * filtered;
414 power /= spec.channels;
415 const auto position = first + f;
416 auto &bin = micro[
static_cast<std::size_t
>(position / microHop)];
417 bin.power += power / microHop;
419 highPower[
static_cast<std::size_t
>(position / microHop)] +=
420 high / spec.channels / microHop;
421 if (power > bin.peakPower)
423 bin.peakPower = power;
424 bin.peakFrame = position;
431 if (fingerprint != analysis.energy_.fingerprint())
433 for (
int f = 0; f < size; ++f)
435 if (frame != analysis.featureCount_)
439 [[nodiscard]]
static double meanPower(PowerView micro, std::int64_t begin,
440 std::int64_t end)
noexcept
443 for (
auto i = std::max<std::int64_t>(0, begin);
444 i < std::min<std::int64_t>(
static_cast<std::int64_t
>(micro.size()), end); ++i)
445 sum += micro[
static_cast<std::size_t
>(i)];
446 const double mean = end > begin ? sum /
static_cast<double>(end - begin) : 0;
447 return micro.high.empty() ? std::min(1.0, mean) : mean;
450 [[nodiscard]]
static bool refine(Event &event, std::int64_t reference,
451 PowerView micro,
int microHop,
452 const OfflineAudioSpec &spec,
bool bass)
noexcept
455 static_cast<std::int64_t
>(std::lround((bass ? 0.045 : 0.025) * spec.sampleRate));
456 if (reference < -radius || reference >= spec.frames + radius)
458 const auto from = std::max<std::int64_t>(0, reference - radius) / microHop;
459 const auto to = std::min<std::int64_t>(spec.frames - 1, reference + radius) / microHop;
460 double best = 0, before = 0, after = 0;
461 std::int64_t selected = from;
462 for (
auto i = from; i <= to; ++i)
464 const double left = meanPower(micro, i - 3, i);
465 const double right = meanPower(micro, i, i + 3);
466 if (right - left > best)
474 if (!(best > 1e-12) || best < 0.12 * after)
484 const double baseline = maximumPower(micro, from - 40, from);
486 for (
auto i = from; i <= to; ++i)
487 maximum = std::max(maximum, meanPower(micro, i, i + 3));
488 const double crossing = baseline + 0.08 * std::max(0.0, maximum - baseline);
490 for (
auto i = from; i <= to; ++i)
492 const double broadLeft = maximumPower(micro, i - 42, i - 2);
493 const double broadRight = maximumPower(micro, i, i + 40);
494 const double left = meanPower(micro, i - 3, i);
495 const double right = meanPower(micro, i, i + 3);
496 if (right > crossing && right > left && broadRight - broadLeft >= 0.4 * broadRight)
514 const double quiet = after * 1e-6;
515 const auto earliest = std::max<std::int64_t>(2, selected - 40);
516 for (
auto i = selected - 1; i >= earliest; --i)
517 if (micro[
static_cast<std::size_t
>(i)] <= quiet &&
518 micro[
static_cast<std::size_t
>(i - 1)] <= quiet &&
519 micro[
static_cast<std::size_t
>(i - 2)] <= quiet)
522 while (onset < selected && micro[
static_cast<std::size_t
>(onset)] <= quiet)
524 if (onset < selected)
527 before = meanPower(micro, onset - 3, onset);
528 after = meanPower(micro, onset, onset + 3);
529 best = after - before;
534 event.contrast = best / std::max(1e-30, before + after);
535 const double floor = before + 0.05 * best;
536 auto begin = selected;
537 while (begin > 0 && begin > selected - 5 &&
538 micro[
static_cast<std::size_t
>(begin - 1)] > floor)
540 while (begin < to && micro[
static_cast<std::size_t
>(begin)] <= floor)
546 const double contextBefore = meanPower(micro, begin - 12, begin - 2);
547 const auto availableEnd =
static_cast<std::int64_t
>(micro.size());
548 const double contextAfter = meanPower(micro, begin, std::min(begin + 10, availableEnd));
549 if (contextAfter - contextBefore < 0.08 * contextAfter)
551 const double broadBefore = meanPower(micro, begin - 22, begin - 2);
552 const double broadAfter = meanPower(micro, begin, std::min(begin + 20, availableEnd));
553 if (broadAfter - broadBefore < 0.08 * broadAfter)
555 const auto &bin = micro.source[
static_cast<std::size_t
>(begin)];
556 event.begin = begin * microHop;
558 if (micro.high.empty() && bin.peakPower > 0.95 * bin.power * microHop)
559 event.begin = bin.peakFrame;
560 event.peak =
event.begin;
561 event.end =
event.begin + 1;
565 [[nodiscard]]
static double maximumPower(PowerView micro, std::int64_t begin,
566 std::int64_t end)
noexcept
569 for (
auto i = begin; i < end; ++i)
570 maximum = std::max(maximum, meanPower(micro, i, i + 3));
574 static void followBody(Event &event,
const Event *next, std::span<const MicroBin> micro,
575 std::span<const double> highPower,
576 int microHop,
const OfflineAudioSpec &spec,
double scale)
noexcept
578 const auto begin =
event.begin / microHop;
580 std::max<std::int64_t>(1,
static_cast<std::int64_t
>(0.25 * spec.sampleRate));
581 const auto limitFrame =
582 std::min(spec.frames, event.begin + std::min(maximum, spec.frames - event.begin));
583 const auto horizon = (next ? std::min(limitFrame, next->begin) : limitFrame);
584 const auto stop = std::max(begin + 1, (horizon - 1) / microHop + 1);
585 const auto tailBegin = std::max(begin, stop - 25);
586 const PowerView body(micro, event.bandRefined ? highPower : std::span<const double>{});
587 const double sustain = meanPower(body, tailBegin, stop);
589 auto crestBin = begin;
590 for (
auto i = begin; i < std::min(stop, begin + 40); ++i)
592 const double power = meanPower(body, i - 1, i + 2);
600 if (crest > 1.25 * sustain)
602 const double floor = sustain + 0.1 * (crest - sustain);
603 for (
auto i = crestBin + 1; i < stop; ++i)
604 if (meanPower(body, i - 1, i + 2) <= floor && meanPower(body, i, i + 3) <= floor)
611 end = std::min(stop, crestBin + 2);
612 event.endLimited = end == stop && horizon < spec.frames;
613 event.overlapsNext =
event.endLimited && next && horizon == next->begin;
614 event.end = std::min(horizon, std::max(event.begin + 1, end * microHop));
615 event.sustainRms = std::sqrt(meanPower(micro, tailBegin, stop)) * scale;
617 for (
auto i = begin; i < stop && i * microHop <
event.end; ++i)
619 const auto &bin = micro[
static_cast<std::size_t
>(i)];
620 if (bin.peakPower > peak && bin.peakFrame >= event.begin && bin.peakFrame < event.end)
622 peak = bin.peakPower;
623 event.peak = bin.peakFrame;
626 event.peakAmplitude = std::sqrt(peak) * scale;
629 static void buildEvents(Analysis &analysis, std::span<const FeatureFrame> features,
630 std::span<const MicroBin> micro, std::span<const double> highPower,
631 std::span<const bool> highEvidence,
int microHop,
bool pulse,
632 detail::OfflineSession &job)
634 const auto spec = analysis.energy_.getSpec();
636 for (
const auto &f : features)
637 maximum = std::max(maximum, pulse ? f.pulse : f.attack);
638 if (!(maximum > T(0)))
640 const auto value = [&](std::size_t i) {
641 return (pulse ? features[i].pulse : features[i].attack) / maximum;
644 static_cast<std::size_t
>(std::lround(0.1 * spec.sampleRate / analysis.hop_));
645 const auto gap =
static_cast<std::size_t
>(
646 std::lround((pulse ? 0.05 : 0.03) * spec.sampleRate / analysis.hop_));
647 const auto gapFrames =
648 static_cast<std::int64_t
>(std::llround((pulse ? 0.05 : 0.03) * spec.sampleRate));
649 const typename detail::OnsetPeakPicker<T>::Windows windows{
650 pulse ? 1u : 6u, pulse ? 1u : 6u, average, average, gap > 0 ? gap - 1 : 0};
651 const auto offset =
static_cast<std::int64_t
>(std::lround(0.34 * analysis.frameSize_));
652 const auto visitSpectral = [&](
auto emit) {
653 detail::OnsetPeakPicker<T> picker;
654 std::int64_t last = -1;
655 for (std::size_t i = 0; i < features.size(); ++i)
657 if ((i & 1023u) == 0)
659 static_cast<std::int64_t
>(features.size()));
663 if (pulse && i > 0 && features[i].windowPower <= features[i - 1].windowPower)
667 if (!pulse && analysis.featureCenter(i) + offset >= spec.frames)
669 if (!picker.accept(i, features.size(), value, windows, T(0.06)))
674 const bool bass = features[i].lowEnergyShare > 0.8;
675 if (!refine(event, analysis.featureCenter(i) + offset, micro, microHop, spec, bass))
677 if (pulse || highPower.empty() ||
678 !refine(event, analysis.featureCenter(i) + offset,
679 PowerView(micro, highPower), microHop, spec,
false))
684 const auto evidence =
static_cast<std::size_t
>(std::clamp<std::int64_t>(
685 (event.begin + analysis.frameSize_ / 2 + analysis.hop_ / 2) /
687 0,
static_cast<std::int64_t
>(features.size() - 1)));
688 if (!highEvidence[evidence])
690 event.bandRefined =
true;
692 if (last >= 0 && event.begin - last < gapFrames)
695 static_cast<double>((pulse ? features[i].pulse : features[i].attack) / maximum);
700 const auto visit = [&](
auto emit) {
711 std::size_t nextBin = 0;
713 bool hasRestart =
false;
714 const auto seekRestart = [&] {
716 while (nextBin < micro.size())
718 const auto at =
static_cast<std::int64_t
>(nextBin++);
719 if (micro[
static_cast<std::size_t
>(at)].power <= 1e-12)
721 const auto &bin = micro[
static_cast<std::size_t
>(at)];
722 const bool impulse = bin.peakPower > .95 * bin.power * microHop;
724 if (at < 8 && !impulse)
727 for (
auto j = std::max<std::int64_t>(0, at - 8); j < at; ++j)
728 before = std::max(before, micro[
static_cast<std::size_t
>(j)].power);
729 const double after = meanPower(micro, at, at + 3);
730 if (before > after * .001)
733 restart.begin = bin.peakFrame;
734 restart.peak = restart.begin;
735 restart.end = restart.begin + 1;
736 restart.strength = std::sqrt(after);
737 restart.contrast = (after - before) / (after + before);
738 restart.energyRestart =
true;
743 std::int64_t last = -1;
744 const auto sameAttack =
static_cast<std::int64_t
>(std::llround(.006 * spec.sampleRate));
745 const auto append = [&](Event event) {
746 if (last < 0 || event.begin - last >= gapFrames)
753 visitSpectral([&](Event event) {
754 while (hasRestart && restart.begin < event.begin - sameAttack)
759 if (hasRestart && restart.begin <= event.begin + sameAttack)
773 std::size_t count = 0;
774 visit([&](Event) { ++count; });
777 auto events = job.allocate<Event>(count);
779 visit([&](Event event) { events[at++] = event; });
780 for (std::size_t i = 0; i < count; ++i)
782 followBody(events[i], i + 1 < count ? &events[i + 1] : nullptr, micro, highPower,
783 microHop, spec, analysis.energy_.samplePeak());
785 static_cast<std::int64_t
>(count));
789 analysis.pulses_ = std::move(events);
790 analysis.pulseCount_ = count;
794 analysis.attacks_ = std::move(events);
795 analysis.attackCount_ = count;