Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# SPDX-License-Identifier: CECILL-2.1
"""
FastDataset(supervisions, recordings, partition)
Constructor for dataset represented as JSONL files (a.k.a. manifests).
"""
struct SpeechDataset <: MLUtils.AbstractDataContainer
idxs::Vector{AbstractString}
superivions::Dict{AbstractString, Supervision}
recordings::Dict{AbstractString, Recording}
partition::Symbol
end
"""
dataset(manifestroot, subset)
Load `SpeechDataset` from manifest files stored in `manifestroot`.
Partition is specified by `subset`, e.g. `:train`, `:test`.
Each item of the dataset is a nested tuple `((samples, sampling_rate), Supervision.data)`.
See also [`Supervision`](@ref).
# Examples
```julia-repl
julia> ds = dataset("./manifests", :train)
SpeechDataset(
...
)
julia> ds[1]
(
(samples=[...], sampling_rate=16_000),
Dict(
"text" => "Supervision text here"
)
)
```
"""
function dataset(manifestroot::AbstractString, subset)
supervisions = load(Supervision, manifestroot, subset)
recordings = load(Recording, manifestroot, subset)
idxs = collect(keys(supervisions))
SpeechDataset(idxs, supervisions, recordings, Symbol(subset))
end
function Base.getindex(d::SpeechDataset, key::AbstractString)
sup = d.superivions[key]
rec = d.recordings[sup.recording_id]
samples, sr = load(rec, sup)
(samples=samples, sampling_rate=sr), sup.data
end
Base.getindex(d::SpeechDataset, idx::Integer) = getindex(d, d.idxs[idx])
# Fix1 -> partial funcion with fixed 1st argument
Base.getindex(d::SpeechDataset, idxs::AbstractVector) = map(Base.Fix1(getindex, d), idxs)
Base.length(d::SpeechDataset) = length(d.idxs)