Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • fast/speechdatasets.jl
  • PTAL/Datasets/SpeechDatasets.jl
2 results
Show changes
Commits on Source (4)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 176 additions and 0 deletions
dev/.documenter-siteinfo.json 0 → 100644
View file @ a28aaed1
{"documenter":{"julia_version":"1.9.4","generation_timestamp":"2024-09-25T15:58:45","documenter_version":"1.7.0"}}
\ No newline at end of file
dev/assets/documenter.js 0 → 100644
View file @ a28aaed1
This diff is collapsed.
dev/assets/logo.svg 0 → 100644
View file @ a28aaed1
<svg version="1.1" width="200" height="200" xmlns="http://www.w3.org/2000/svg">
<ellipse id="petal" cx="52.5" cy="100" rx="42.5" ry="30"
stroke="black" stroke-opacity="0"
fill-opacity="1" fill="#08d87b"/>
<use href="#petal" transform="rotate(45, 100, 100)"/>
<use href="#petal" transform="rotate(90, 100, 100)"/>
<use href="#petal" transform="rotate(135, 100, 100)"/>
<use href="#petal" transform="rotate(180, 100, 100)"/>
<use href="#petal" transform="rotate(225, 100, 100)"/>
<use href="#petal" transform="rotate(270, 100, 100)"/>
<use href="#petal" transform="rotate(315, 100, 100)"/>
</svg>
\ No newline at end of file
dev/assets/themes/catppuccin-frappe.css 0 → 100644
View file @ a28aaed1
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dev/assets/themes/catppuccin-latte.css 0 → 100644
View file @ a28aaed1
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dev/assets/themes/catppuccin-macchiato.css 0 → 100644
View file @ a28aaed1
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dev/assets/themes/catppuccin-mocha.css 0 → 100644
View file @ a28aaed1
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dev/assets/themes/documenter-dark.css 0 → 100644
View file @ a28aaed1
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dev/assets/themes/documenter-light.css 0 → 100644
View file @ a28aaed1
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dev/assets/themeswap.js 0 → 100644
View file @ a28aaed1
// Small function to quickly swap out themes. Gets put into the <head> tag..
function set_theme_from_local_storage() {
// Initialize the theme to null, which means default
var theme = null;
// If the browser supports the localstorage and is not disabled then try to get the
// documenter theme
if (window.localStorage != null) {
// Get the user-picked theme from localStorage. May be `null`, which means the default
// theme.
theme = window.localStorage.getItem("documenter-theme");
}
// Check if the users preference is for dark color scheme
var darkPreference =
window.matchMedia("(prefers-color-scheme: dark)").matches === true;
// Initialize a few variables for the loop:
//
// - active: will contain the index of the theme that should be active. Note that there
// is no guarantee that localStorage contains sane values. If `active` stays `null`
// we either could not find the theme or it is the default (primary) theme anyway.
// Either way, we then need to stick to the primary theme.
//
// - disabled: style sheets that should be disabled (i.e. all the theme style sheets
// that are not the currently active theme)
var active = null;
var disabled = [];
var primaryLightTheme = null;
var primaryDarkTheme = null;
for (var i = 0; i < document.styleSheets.length; i++) {
var ss = document.styleSheets[i];
// The <link> tag of each style sheet is expected to have a data-theme-name attribute
// which must contain the name of the theme. The names in localStorage much match this.
var themename = ss.ownerNode.getAttribute("data-theme-name");
// attribute not set => non-theme stylesheet => ignore
if (themename === null) continue;
// To distinguish the default (primary) theme, it needs to have the data-theme-primary
// attribute set.
if (ss.ownerNode.getAttribute("data-theme-primary") !== null) {
primaryLightTheme = themename;
}
// Check if the theme is primary dark theme so that we could store its name in darkTheme
if (ss.ownerNode.getAttribute("data-theme-primary-dark") !== null) {
primaryDarkTheme = themename;
}
// If we find a matching theme (and it's not the default), we'll set active to non-null
if (themename === theme) active = i;
// Store the style sheets of inactive themes so that we could disable them
if (themename !== theme) disabled.push(ss);
}
var activeTheme = null;
if (active !== null) {
// If we did find an active theme, we'll (1) add the theme--$(theme) class to <html>
document.getElementsByTagName("html")[0].className = "theme--" + theme;
activeTheme = theme;
} else {
// If we did _not_ find an active theme, then we need to fall back to the primary theme
// which can either be dark or light, depending on the user's OS preference.
var activeTheme = darkPreference ? primaryDarkTheme : primaryLightTheme;
// In case it somehow happens that the relevant primary theme was not found in the
// preceding loop, we abort without doing anything.
if (activeTheme === null) {
console.error("Unable to determine primary theme.");
return;
}
// When switching to the primary light theme, then we must not have a class name
// for the <html> tag. That's only for non-primary or the primary dark theme.
if (darkPreference) {
document.getElementsByTagName("html")[0].className =
"theme--" + activeTheme;
} else {
document.getElementsByTagName("html")[0].className = "";
}
}
for (var i = 0; i < document.styleSheets.length; i++) {
var ss = document.styleSheets[i];
// The <link> tag of each style sheet is expected to have a data-theme-name attribute
// which must contain the name of the theme. The names in localStorage much match this.
var themename = ss.ownerNode.getAttribute("data-theme-name");
// attribute not set => non-theme stylesheet => ignore
if (themename === null) continue;
// we'll disable all the stylesheets, except for the active one
ss.disabled = !(themename == activeTheme);
}
}
set_theme_from_local_storage();
dev/assets/warner.js 0 → 100644
View file @ a28aaed1
function maybeAddWarning() {
// DOCUMENTER_NEWEST is defined in versions.js, DOCUMENTER_CURRENT_VERSION and DOCUMENTER_STABLE
// in siteinfo.js.
// If either of these are undefined something went horribly wrong, so we abort.
if (
window.DOCUMENTER_NEWEST === undefined ||
window.DOCUMENTER_CURRENT_VERSION === undefined ||
window.DOCUMENTER_STABLE === undefined
) {
return;
}
// Current version is not a version number, so we can't tell if it's the newest version. Abort.
if (!/v(\d+\.)*\d+/.test(window.DOCUMENTER_CURRENT_VERSION)) {
return;
}
// Current version is newest version, so no need to add a warning.
if (window.DOCUMENTER_NEWEST === window.DOCUMENTER_CURRENT_VERSION) {
return;
}
// Add a noindex meta tag (unless one exists) so that search engines don't index this version of the docs.
if (document.body.querySelector('meta[name="robots"]') === null) {
const meta = document.createElement("meta");
meta.name = "robots";
meta.content = "noindex";
document.getElementsByTagName("head")[0].appendChild(meta);
}
const div = document.createElement("div");
div.classList.add("outdated-warning-overlay");
const closer = document.createElement("button");
closer.classList.add("outdated-warning-closer", "delete");
closer.addEventListener("click", function () {
document.body.removeChild(div);
});
const href = window.documenterBaseURL + "/../" + window.DOCUMENTER_STABLE;
div.innerHTML =
'This documentation is not for the latest stable release, but for either the development version or an older release.<br><a href="' +
href +
'">Click here to go to the documentation for the latest stable release.</a>';
div.appendChild(closer);
document.body.appendChild(div);
}
if (document.readyState === "loading") {
document.addEventListener("DOMContentLoaded", maybeAddWarning);
} else {
maybeAddWarning();
}
dev/datasets/index.html 0 → 100644
View file @ a28aaed1
<!DOCTYPE html>
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Supported Datasets · SpeechDatasets</title><meta name="title" content="Supported Datasets · SpeechDatasets"/><meta property="og:title" content="Supported Datasets · SpeechDatasets"/><meta property="twitter:title" content="Supported Datasets · SpeechDatasets"/><meta name="description" content="Documentation for SpeechDatasets."/><meta property="og:description" content="Documentation for SpeechDatasets."/><meta property="twitter:description" content="Documentation for SpeechDatasets."/><script data-outdated-warner src="../assets/warner.js"></script><link href="https://cdnjs.cloudflare.com/ajax/libs/lato-font/3.0.0/css/lato-font.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/juliamono/0.050/juliamono.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.8/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../search_index.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-mocha.css" data-theme-name="catppuccin-mocha"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-macchiato.css" data-theme-name="catppuccin-macchiato"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-frappe.css" data-theme-name="catppuccin-frappe"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-latte.css" data-theme-name="catppuccin-latte"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark" data-theme-primary-dark/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><a class="docs-logo" href="../"><img src="../assets/logo.svg" alt="SpeechDatasets logo"/></a><div class="docs-package-name"><span class="docs-autofit"><a href="../">SpeechDatasets</a></span></div><button class="docs-search-query input is-rounded is-small is-clickable my-2 mx-auto py-1 px-2" id="documenter-search-query">Search docs (Ctrl + /)</button><ul class="docs-menu"><li><a class="tocitem" href="../">SpeechDatasets.jl</a></li><li class="is-active"><a class="tocitem" href>Supported Datasets</a></li><li><a class="tocitem" href="../installation/">Installation</a></li><li><a class="tocitem" href="../newdataset/">Add a new dataset</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><a class="docs-sidebar-button docs-navbar-link fa-solid fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Supported Datasets</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Supported Datasets</a></li></ul></nav><div class="docs-right"><a class="docs-navbar-link" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl" title="View the repository"><span class="docs-icon fa-brands"></span><span class="docs-label is-hidden-touch">Repository</span></a><a class="docs-navbar-link" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/main/docs/src/datasets.md" title="Edit source"><span class="docs-icon fa-solid"></span></a><a class="docs-settings-button docs-navbar-link fa-solid fa-gear" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-article-toggle-button fa-solid fa-chevron-up" id="documenter-article-toggle-button" href="javascript:;" title="Collapse all docstrings"></a></div></header><article class="content" id="documenter-page"><h1 id="Supported-Datasets"><a class="docs-heading-anchor" href="#Supported-Datasets">Supported Datasets</a><a id="Supported-Datasets-1"></a><a class="docs-heading-anchor-permalink" href="#Supported-Datasets" title="Permalink"></a></h1><h2>AVID</h2><p><img src="https://img.shields.io/badge/License-CC BY 4.0-lightblue" alt="License"/> <img src="https://img.shields.io/badge/Lang-eng-lightgreen" alt="Language"/></p><p>Aalto Vocal Intensity Database includes speech and EGG produced by 50 speakers (25 males, 25 females) who varied their vocal intensity in four categories (soft, normal, loud, and very loud).</p><p><a href="https://zenodo.org/records/10524873">Source</a></p><h3>Authors</h3><p>Manila Kodali, Paavo Alku, Sudarsana Reddy Kadiri</p><hr/><h2>INA Diachrony</h2><p><img src="https://img.shields.io/badge/License-proprietary-red" alt="License"/> <img src="https://img.shields.io/badge/Lang-fra-lightgreen" alt="Language"/></p><p>Voice recordings and transcriptions sorted by time period, sex and speaker.</p><h3>Keyword arguments</h3><pre><code class="language-julia hljs">(ina_csv_dir = nothing,)</code></pre><hr/><h2>Mini LibriSpeech</h2><p><img src="https://img.shields.io/badge/License-CC BY 4.0-lightblue" alt="License"/> <img src="https://img.shields.io/badge/Lang-eng-lightgreen" alt="Language"/></p><p>Subset of LibriSpeech corpus for purpose of regression testing.</p><p><a href="https://www.openslr.org/31/">Source</a></p><h3>Authors</h3><p>Vassil Panayotov, Daniel Povey</p><h3>Subsets</h3><p>train, dev</p><hr/><h2>Multilingual LibriSpeech</h2><p><img src="https://img.shields.io/badge/License-CC BY 4.0-lightblue" alt="License"/> <img src="https://img.shields.io/badge/Lang-eng-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-fra-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-prt-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-esp-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-deu-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-nld-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-ita-lightgreen" alt="Language"/> <img src="https://img.shields.io/badge/Lang-pol-lightgreen" alt="Language"/></p><p>Multilingual LibriSpeech (MLS) dataset is a large multilingual corpus suitable for speech research. The dataset is derived from read audiobooks from LibriVox and consists of 8 languages - English, German, Dutch, Spanish, French, Italian, Portuguese, Polish</p><p><a href="http://www.openslr.org/94">Source</a></p><h3>Authors</h3><p>Vineel Pratap, Qiantong Xu, Anuroop Sriram, Gabriel Synnaeve, Ronan Collobert</p><h3>Subsets</h3><p>train, dev, test</p><h3>Keyword arguments</h3><pre><code class="language-julia hljs">(lang = &quot;eng&quot;,)</code></pre><hr/><h2>TIMIT</h2><p><img src="https://img.shields.io/badge/License-LDC User Agreement for Non--Members-lightblue" alt="License"/> <img src="https://img.shields.io/badge/Lang-eng-lightgreen" alt="Language"/></p><p>The TIMIT corpus of read speech has been designed to provide speech data for the acquisition of acoustic-phonetic knowledge and for the development and evaluation of automatic speech recognition systems.</p><p><a href="https://catalog.ldc.upenn.edu/LDC93S1">Source</a></p><h3>Authors</h3><p>John S. Garofolo, Lori F. Lamel, William M. Fisher, Jonathan G. Fiscus, David S. Pallett, Nancy L. Dahlgren, Victor Zue</p><h3>Subsets</h3><p>train, dev, test</p><h3>Keyword arguments</h3><pre><code class="language-julia hljs">(formantsdir = nothing, audio_fmt = &quot;SPHERE&quot;)</code></pre><hr/><h2>Speech2Tex</h2><p><img src="https://img.shields.io/badge/License-proprietary-red" alt="License"/> <img src="https://img.shields.io/badge/Lang-fra-lightgreen" alt="Language"/></p><p>Recordings of read equations, literal transcriptions and latex transcriptions.</p><h3>Authors</h3><p>Lorenzo Brucato</p><hr/></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../">« SpeechDatasets.jl</a><a class="docs-footer-nextpage" href="../installation/">Installation »</a><div class="flexbox-break"></div><p class="footer-message">Powered by <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> and the <a href="https://julialang.org/">Julia Programming Language</a>.</p></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="auto">Automatic (OS)</option><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option><option value="catppuccin-latte">catppuccin-latte</option><option value="catppuccin-frappe">catppuccin-frappe</option><option value="catppuccin-macchiato">catppuccin-macchiato</option><option value="catppuccin-mocha">catppuccin-mocha</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> version 1.7.0 on <span class="colophon-date" title="Wednesday 25 September 2024 15:58">Wednesday 25 September 2024</span>. Using Julia version 1.9.4.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
dev/index.html 0 → 100644
View file @ a28aaed1
This diff is collapsed.
dev/installation/index.html 0 → 100644
View file @ a28aaed1
<!DOCTYPE html>
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Installation · SpeechDatasets</title><meta name="title" content="Installation · SpeechDatasets"/><meta property="og:title" content="Installation · SpeechDatasets"/><meta property="twitter:title" content="Installation · SpeechDatasets"/><meta name="description" content="Documentation for SpeechDatasets."/><meta property="og:description" content="Documentation for SpeechDatasets."/><meta property="twitter:description" content="Documentation for SpeechDatasets."/><script data-outdated-warner src="../assets/warner.js"></script><link href="https://cdnjs.cloudflare.com/ajax/libs/lato-font/3.0.0/css/lato-font.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/juliamono/0.050/juliamono.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.8/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../search_index.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-mocha.css" data-theme-name="catppuccin-mocha"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-macchiato.css" data-theme-name="catppuccin-macchiato"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-frappe.css" data-theme-name="catppuccin-frappe"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-latte.css" data-theme-name="catppuccin-latte"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark" data-theme-primary-dark/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><a class="docs-logo" href="../"><img src="../assets/logo.svg" alt="SpeechDatasets logo"/></a><div class="docs-package-name"><span class="docs-autofit"><a href="../">SpeechDatasets</a></span></div><button class="docs-search-query input is-rounded is-small is-clickable my-2 mx-auto py-1 px-2" id="documenter-search-query">Search docs (Ctrl + /)</button><ul class="docs-menu"><li><a class="tocitem" href="../">SpeechDatasets.jl</a></li><li><a class="tocitem" href="../datasets/">Supported Datasets</a></li><li class="is-active"><a class="tocitem" href>Installation</a></li><li><a class="tocitem" href="../newdataset/">Add a new dataset</a></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><a class="docs-sidebar-button docs-navbar-link fa-solid fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Installation</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Installation</a></li></ul></nav><div class="docs-right"><a class="docs-navbar-link" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl" title="View the repository"><span class="docs-icon fa-brands"></span><span class="docs-label is-hidden-touch">Repository</span></a><a class="docs-navbar-link" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/main/docs/src/installation.md" title="Edit source"><span class="docs-icon fa-solid"></span></a><a class="docs-settings-button docs-navbar-link fa-solid fa-gear" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-article-toggle-button fa-solid fa-chevron-up" id="documenter-article-toggle-button" href="javascript:;" title="Collapse all docstrings"></a></div></header><article class="content" id="documenter-page"><h1 id="Installation"><a class="docs-heading-anchor" href="#Installation">Installation</a><a id="Installation-1"></a><a class="docs-heading-anchor-permalink" href="#Installation" title="Permalink"></a></h1><p>This package is part of the PTAL tool collection and requires the <a href="https://gitlab.lisn.upsaclay.fr/ptal/registry">PTAL registry</a> to be installed.</p><p>To add this registry to your Julia installation type <code>]</code> to enter the package mode of the REPL and then type:</p><pre><code class="nohighlight hljs">pkg&gt; registry add &quot;https://gitlab.lisn.upsaclay.fr/PTAL/Registry&quot;</code></pre><p>Once the registry has been added, SpeechDatasets can be installed with the Julia package manager by typing in Pkg REPL mode</p><pre><code class="nohighlight hljs">pkg&gt; add SpeechDatasets</code></pre></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../datasets/">« Supported Datasets</a><a class="docs-footer-nextpage" href="../newdataset/">Add a new dataset »</a><div class="flexbox-break"></div><p class="footer-message">Powered by <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> and the <a href="https://julialang.org/">Julia Programming Language</a>.</p></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="auto">Automatic (OS)</option><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option><option value="catppuccin-latte">catppuccin-latte</option><option value="catppuccin-frappe">catppuccin-frappe</option><option value="catppuccin-macchiato">catppuccin-macchiato</option><option value="catppuccin-mocha">catppuccin-mocha</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> version 1.7.0 on <span class="colophon-date" title="Wednesday 25 September 2024 15:58">Wednesday 25 September 2024</span>. Using Julia version 1.9.4.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
dev/newdataset/index.html 0 → 100644
View file @ a28aaed1
<!DOCTYPE html>
<html lang="en"><head><meta charset="UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><title>Add a new dataset · SpeechDatasets</title><meta name="title" content="Add a new dataset · SpeechDatasets"/><meta property="og:title" content="Add a new dataset · SpeechDatasets"/><meta property="twitter:title" content="Add a new dataset · SpeechDatasets"/><meta name="description" content="Documentation for SpeechDatasets."/><meta property="og:description" content="Documentation for SpeechDatasets."/><meta property="twitter:description" content="Documentation for SpeechDatasets."/><script data-outdated-warner src="../assets/warner.js"></script><link href="https://cdnjs.cloudflare.com/ajax/libs/lato-font/3.0.0/css/lato-font.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/juliamono/0.050/juliamono.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/fontawesome.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/solid.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/brands.min.css" rel="stylesheet" type="text/css"/><link href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.8/katex.min.css" rel="stylesheet" type="text/css"/><script>documenterBaseURL=".."</script><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js" data-main="../assets/documenter.js"></script><script src="../search_index.js"></script><script src="../siteinfo.js"></script><script src="../../versions.js"></script><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-mocha.css" data-theme-name="catppuccin-mocha"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-macchiato.css" data-theme-name="catppuccin-macchiato"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-frappe.css" data-theme-name="catppuccin-frappe"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/catppuccin-latte.css" data-theme-name="catppuccin-latte"/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-dark.css" data-theme-name="documenter-dark" data-theme-primary-dark/><link class="docs-theme-link" rel="stylesheet" type="text/css" href="../assets/themes/documenter-light.css" data-theme-name="documenter-light" data-theme-primary/><script src="../assets/themeswap.js"></script></head><body><div id="documenter"><nav class="docs-sidebar"><a class="docs-logo" href="../"><img src="../assets/logo.svg" alt="SpeechDatasets logo"/></a><div class="docs-package-name"><span class="docs-autofit"><a href="../">SpeechDatasets</a></span></div><button class="docs-search-query input is-rounded is-small is-clickable my-2 mx-auto py-1 px-2" id="documenter-search-query">Search docs (Ctrl + /)</button><ul class="docs-menu"><li><a class="tocitem" href="../">SpeechDatasets.jl</a></li><li><a class="tocitem" href="../datasets/">Supported Datasets</a></li><li><a class="tocitem" href="../installation/">Installation</a></li><li class="is-active"><a class="tocitem" href>Add a new dataset</a><ul class="internal"><li><a class="tocitem" href="#DatasetBuilder-and-utilities"><span>DatasetBuilder and utilities</span></a></li></ul></li></ul><div class="docs-version-selector field has-addons"><div class="control"><span class="docs-label button is-static is-size-7">Version</span></div><div class="docs-selector control is-expanded"><div class="select is-fullwidth is-size-7"><select id="documenter-version-selector"></select></div></div></div></nav><div class="docs-main"><header class="docs-navbar"><a class="docs-sidebar-button docs-navbar-link fa-solid fa-bars is-hidden-desktop" id="documenter-sidebar-button" href="#"></a><nav class="breadcrumb"><ul class="is-hidden-mobile"><li class="is-active"><a href>Add a new dataset</a></li></ul><ul class="is-hidden-tablet"><li class="is-active"><a href>Add a new dataset</a></li></ul></nav><div class="docs-right"><a class="docs-navbar-link" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl" title="View the repository"><span class="docs-icon fa-brands"></span><span class="docs-label is-hidden-touch">Repository</span></a><a class="docs-navbar-link" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/main/docs/src/newdataset.md" title="Edit source"><span class="docs-icon fa-solid"></span></a><a class="docs-settings-button docs-navbar-link fa-solid fa-gear" id="documenter-settings-button" href="#" title="Settings"></a><a class="docs-article-toggle-button fa-solid fa-chevron-up" id="documenter-article-toggle-button" href="javascript:;" title="Collapse all docstrings"></a></div></header><article class="content" id="documenter-page"><h1 id="Add-a-new-dataset"><a class="docs-heading-anchor" href="#Add-a-new-dataset">Add a new dataset</a><a id="Add-a-new-dataset-1"></a><a class="docs-heading-anchor-permalink" href="#Add-a-new-dataset" title="Permalink"></a></h1><ol><li><p>Add metadatas in <code>src/corpora/corpora.json</code> </p><p>Example:</p><pre><code class="nohighlight hljs"> {
&quot;name&quot;: &quot;TIMIT&quot;,
&quot;lang&quot;: &quot;eng&quot;,
&quot;license&quot;: &quot;LDC User Agreement for Non-Members&quot;,
&quot;source&quot;: &quot;https://catalog.ldc.upenn.edu/LDC93S1&quot;,
&quot;authors&quot;: [&quot;John S. Garofolo&quot;, &quot;Lori F. Lamel&quot;, &quot;William M. Fisher&quot;, &quot;Jonathan G. Fiscus&quot;, &quot;David S. Pallett&quot;, &quot;Nancy L. Dahlgren&quot;, &quot;Victor Zue&quot;],
&quot;description&quot;: &quot;The TIMIT corpus of read speech has been designed to provide speech data for the acquisition of acoustic-phonetic knowledge and for the development and evaluation of automatic speech recognition systems.&quot;,
&quot;subsets&quot;: [&quot;train&quot;, &quot;dev&quot;, &quot;test&quot;]
},</code></pre></li><li><p>Create a new <code>.jl</code> file in <code>src/corpora</code></p></li><li><p>Add the following line at the beginning of the file: </p><pre><code class="nohighlight hljs"> const &lt;idname&gt; = get_nametype(&lt;dataset name&gt;)</code></pre><ul><li>Replace <code>&lt;idname&gt;</code> with an identifier of your dataset (for example, <code>timit_id</code>).</li><li>Replace <code>&lt;dataset name&gt;</code> with a string containing the name of the dataset (same as referenced in <code>corpora.json</code>).</li></ul></li><li><p>If your dataset is downloadable, you can implement</p><pre><code class="nohighlight hljs"> Base.download(::DatasetBuilder{&lt;idname&gt;}, dir::AbstractString)</code></pre></li><li><p>It is mandatory to implement the <code>prepare()</code> function as such: </p><pre><code class="nohighlight hljs"> prepare(::DatasetBuilder{&lt;idname&gt;}, inputdir, outputdir; &lt;keyword arguments&gt;)</code></pre><p>You can add any keyword argument. This function must create the following files in outputdir:</p><ul><li><code>recordings.jsonl</code></li><li><code>annotations.jsonl</code> or <code>annotations-&lt;subset&gt;.jsonl</code> for each subset</li></ul></li></ol><p>That&#39;s it, you can now use </p><pre><code class="language-julia hljs">dataset(&quot;name&quot;, inputdir, outputdir; &lt;keyword arguments&gt;)</code></pre><h2 id="DatasetBuilder-and-utilities"><a class="docs-heading-anchor" href="#DatasetBuilder-and-utilities">DatasetBuilder and utilities</a><a id="DatasetBuilder-and-utilities-1"></a><a class="docs-heading-anchor-permalink" href="#DatasetBuilder-and-utilities" title="Permalink"></a></h2><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="SpeechDatasets.DatasetBuilder" href="#SpeechDatasets.DatasetBuilder"><code>SpeechDatasets.DatasetBuilder</code></a> — <span class="docstring-category">Type</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">struct DatasetBuilder{name}</code></pre><p>Allow to dispatch main dataset functions (<code>download()</code>, <code>prepare()</code>). </p><p><strong>Parameter</strong></p><ul><li><code>name</code> Dataset identifier</li></ul><p><strong>Fields</strong></p><ul><li><code>kwargs::NamedTuple</code> Keyword arguments supported by the dataset associated to <code>name</code></li></ul></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L1-L8">source</a></section></article><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="SpeechDatasets.DatasetBuilder-Tuple{Symbol}" href="#SpeechDatasets.DatasetBuilder-Tuple{Symbol}"><code>SpeechDatasets.DatasetBuilder</code></a> — <span class="docstring-category">Method</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">DatasetBuilder(name::Symbol)</code></pre><p>Construct a DatasetBuilder for a given name. Implementations for each name are done by calling <a href="#SpeechDatasets.declareBuilder-Tuple{Symbol}"><code>declareBuilder(name)</code></a> (automatically done for each supported name).</p></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L13-L17">source</a></section></article><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="SpeechDatasets.declareBuilder-Tuple{Symbol}" href="#SpeechDatasets.declareBuilder-Tuple{Symbol}"><code>SpeechDatasets.declareBuilder</code></a> — <span class="docstring-category">Method</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">declareBuilder(name::Symbol)</code></pre><p>Declare a functor for a DatasetBuilder of type <code>name</code>.</p><p>A <code>DatasetBuilder{name}</code> object can now be created, and will hold the supported kwargs for the corresponding dataset.</p></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L74-L79">source</a></section></article><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="SpeechDatasets.get_kwargs-Tuple{Function, Tuple}" href="#SpeechDatasets.get_kwargs-Tuple{Function, Tuple}"><code>SpeechDatasets.get_kwargs</code></a> — <span class="docstring-category">Method</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">get_kwargs(func_name::Function, args_types::Tuple)</code></pre><p>Return a <code>NamedTuple</code> containing each supported kwarg and its default value for a given method.</p><p><strong>Arguments</strong></p><ul><li><code>func_name</code> is the name of the function</li><li><code>args_types</code> is a tuple of argument types for the desired method</li></ul></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L20-L26">source</a></section></article><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="SpeechDatasets.get_nametype-Tuple{String}" href="#SpeechDatasets.get_nametype-Tuple{String}"><code>SpeechDatasets.get_nametype</code></a> — <span class="docstring-category">Method</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">get_nametype(name::String)</code></pre><p>Return a symbol corresponding to the name. This symbol is used to identify the dataset.</p></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L56-L59">source</a></section></article><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="Base.download" href="#Base.download"><code>Base.download</code></a> — <span class="docstring-category">Function</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">Base.download(builder::DatasetBuilder{name}, dir::AbstractString)</code></pre><p>Download the dataset identified by <code>name</code> into <code>dir</code>.</p><p>Each dataset has its own implementation if download is supported (for example, a proprietary dataset might not implements download).</p></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L86-L91">source</a></section></article><article class="docstring"><header><a class="docstring-article-toggle-button fa-solid fa-chevron-down" href="javascript:;" title="Collapse docstring"></a><a class="docstring-binding" id="SpeechDatasets.prepare" href="#SpeechDatasets.prepare"><code>SpeechDatasets.prepare</code></a> — <span class="docstring-category">Function</span><span class="is-flex-grow-1 docstring-article-toggle-button" title="Collapse docstring"></span></header><section><div><pre><code class="language-julia hljs">prepare(::DatasetBuilder{name}, inputdir, outputdir; &lt;keyword arguments&gt;)</code></pre><p>Create manifest files into <code>outputdir</code> from dataset in <code>inputdir</code>. </p><p>Each dataset has its own implementation, and can have optional keyword arguments, they can be accessed with <a href="../#SpeechDatasets.get_dataset_kwargs-Tuple{String}"><code>get_dataset_kwargs(name::String)</code></a>.</p><p>Implementing this function is mandatory for a dataset to be compatible with <code>dataset()</code></p></div><a class="docs-sourcelink" target="_blank" href="https://gitlab.lisn.upsaclay.fr/PTAL/Datasets/SpeechDatasets.jl/-/tree/40d3314a92b8d038c9db4d25af9b7333da6afec9/src/builder.jl#L93-L100">source</a></section></article></article><nav class="docs-footer"><a class="docs-footer-prevpage" href="../installation/">« Installation</a><div class="flexbox-break"></div><p class="footer-message">Powered by <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> and the <a href="https://julialang.org/">Julia Programming Language</a>.</p></nav></div><div class="modal" id="documenter-settings"><div class="modal-background"></div><div class="modal-card"><header class="modal-card-head"><p class="modal-card-title">Settings</p><button class="delete"></button></header><section class="modal-card-body"><p><label class="label">Theme</label><div class="select"><select id="documenter-themepicker"><option value="auto">Automatic (OS)</option><option value="documenter-light">documenter-light</option><option value="documenter-dark">documenter-dark</option><option value="catppuccin-latte">catppuccin-latte</option><option value="catppuccin-frappe">catppuccin-frappe</option><option value="catppuccin-macchiato">catppuccin-macchiato</option><option value="catppuccin-mocha">catppuccin-mocha</option></select></div></p><hr/><p>This document was generated with <a href="https://github.com/JuliaDocs/Documenter.jl">Documenter.jl</a> version 1.7.0 on <span class="colophon-date" title="Wednesday 25 September 2024 15:58">Wednesday 25 September 2024</span>. Using Julia version 1.9.4.</p></section><footer class="modal-card-foot"></footer></div></div></div></body></html>
dev/objects.inv 0 → 100644
View file @ a28aaed1
File added
dev/search_index.js 0 → 100644
View file @ a28aaed1
var documenterSearchIndex = {"docs":
[{"location":"newdataset/#Add-a-new-dataset","page":"Add a new dataset","title":"Add a new dataset","text":"","category":"section"},{"location":"newdataset/","page":"Add a new dataset","title":"Add a new dataset","text":"Add metadatas in src/corpora/corpora.json \nExample:\n {\n \"name\": \"TIMIT\",\n \"lang\": \"eng\",\n \"license\": \"LDC User Agreement for Non-Members\",\n \"source\": \"https://catalog.ldc.upenn.edu/LDC93S1\",\n \"authors\": [\"John S. Garofolo\", \"Lori F. Lamel\", \"William M. Fisher\", \"Jonathan G. Fiscus\", \"David S. Pallett\", \"Nancy L. Dahlgren\", \"Victor Zue\"],\n \"description\": \"The TIMIT corpus of read speech has been designed to provide speech data for the acquisition of acoustic-phonetic knowledge and for the development and evaluation of automatic speech recognition systems.\",\n \"subsets\": [\"train\", \"dev\", \"test\"]\n },\nCreate a new .jl file in src/corpora\nAdd the following line at the beginning of the file: \n const <idname> = get_nametype(<dataset name>)\nReplace <idname> with an identifier of your dataset (for example, timit_id).\nReplace <dataset name> with a string containing the name of the dataset (same as referenced in corpora.json).\nIf your dataset is downloadable, you can implement\n Base.download(::DatasetBuilder{<idname>}, dir::AbstractString)\nIt is mandatory to implement the prepare() function as such: \n prepare(::DatasetBuilder{<idname>}, inputdir, outputdir; <keyword arguments>)\nYou can add any keyword argument. This function must create the following files in outputdir:\nrecordings.jsonl\nannotations.jsonl or annotations-<subset>.jsonl for each subset","category":"page"},{"location":"newdataset/","page":"Add a new dataset","title":"Add a new dataset","text":"That's it, you can now use ","category":"page"},{"location":"newdataset/","page":"Add a new dataset","title":"Add a new dataset","text":"dataset(\"name\", inputdir, outputdir; <keyword arguments>)","category":"page"},{"location":"newdataset/#DatasetBuilder-and-utilities","page":"Add a new dataset","title":"DatasetBuilder and utilities","text":"","category":"section"},{"location":"newdataset/","page":"Add a new dataset","title":"Add a new dataset","text":"DatasetBuilder\nDatasetBuilder(name::Symbol)\nSpeechDatasets.declareBuilder(name::Symbol)\nget_kwargs(func_name::Function, args_types::Tuple)\nget_nametype(name::String)\nBase.download\nprepare","category":"page"},{"location":"newdataset/#SpeechDatasets.DatasetBuilder","page":"Add a new dataset","title":"SpeechDatasets.DatasetBuilder","text":"struct DatasetBuilder{name}\n\nAllow to dispatch main dataset functions (download(), prepare()). \n\nParameter\n\nname Dataset identifier\n\nFields\n\nkwargs::NamedTuple Keyword arguments supported by the dataset associated to name\n\n\n\n\n\n","category":"type"},{"location":"newdataset/#SpeechDatasets.DatasetBuilder-Tuple{Symbol}","page":"Add a new dataset","title":"SpeechDatasets.DatasetBuilder","text":"DatasetBuilder(name::Symbol)\n\nConstruct a DatasetBuilder for a given name. Implementations for each name are done by calling declareBuilder(name) (automatically done for each supported name).\n\n\n\n\n\n","category":"method"},{"location":"newdataset/#SpeechDatasets.declareBuilder-Tuple{Symbol}","page":"Add a new dataset","title":"SpeechDatasets.declareBuilder","text":"declareBuilder(name::Symbol)\n\nDeclare a functor for a DatasetBuilder of type name.\n\nA DatasetBuilder{name} object can now be created, and will hold the supported kwargs for the corresponding dataset.\n\n\n\n\n\n","category":"method"},{"location":"newdataset/#SpeechDatasets.get_kwargs-Tuple{Function, Tuple}","page":"Add a new dataset","title":"SpeechDatasets.get_kwargs","text":"get_kwargs(func_name::Function, args_types::Tuple)\n\nReturn a NamedTuple containing each supported kwarg and its default value for a given method.\n\nArguments\n\nfunc_name is the name of the function\nargs_types is a tuple of argument types for the desired method\n\n\n\n\n\n","category":"method"},{"location":"newdataset/#SpeechDatasets.get_nametype-Tuple{String}","page":"Add a new dataset","title":"SpeechDatasets.get_nametype","text":"get_nametype(name::String)\n\nReturn a symbol corresponding to the name. This symbol is used to identify the dataset.\n\n\n\n\n\n","category":"method"},{"location":"newdataset/#Base.download","page":"Add a new dataset","title":"Base.download","text":"Base.download(builder::DatasetBuilder{name}, dir::AbstractString)\n\nDownload the dataset identified by name into dir.\n\nEach dataset has its own implementation if download is supported (for example, a proprietary dataset might not implements download).\n\n\n\n\n\n","category":"function"},{"location":"newdataset/#SpeechDatasets.prepare","page":"Add a new dataset","title":"SpeechDatasets.prepare","text":"prepare(::DatasetBuilder{name}, inputdir, outputdir; <keyword arguments>)\n\nCreate manifest files into outputdir from dataset in inputdir. \n\nEach dataset has its own implementation, and can have optional keyword arguments, they can be accessed with get_dataset_kwargs(name::String).\n\nImplementing this function is mandatory for a dataset to be compatible with dataset()\n\n\n\n\n\n","category":"function"},{"location":"installation/#Installation","page":"Installation","title":"Installation","text":"","category":"section"},{"location":"installation/","page":"Installation","title":"Installation","text":"This package is part of the PTAL tool collection and requires the PTAL registry to be installed.","category":"page"},{"location":"installation/","page":"Installation","title":"Installation","text":"To add this registry to your Julia installation type ] to enter the package mode of the REPL and then type:","category":"page"},{"location":"installation/","page":"Installation","title":"Installation","text":"pkg> registry add \"https://gitlab.lisn.upsaclay.fr/PTAL/Registry\"","category":"page"},{"location":"installation/","page":"Installation","title":"Installation","text":"Once the registry has been added, SpeechDatasets can be installed with the Julia package manager by typing in Pkg REPL mode","category":"page"},{"location":"installation/","page":"Installation","title":"Installation","text":"pkg> add SpeechDatasets","category":"page"},{"location":"#SpeechDatasets.jl","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"","category":"section"},{"location":"#Contents","page":"SpeechDatasets.jl","title":"Contents","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"Depth = 3","category":"page"},{"location":"#Example","page":"SpeechDatasets.jl","title":"Example","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"using SpeechDatasets\nds = dataset(\"Mini LibriSpeech\", \"path/to/minils\", \"minils_output\")\ntypeof(ds[26])","category":"page"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"println(\"Tuple{Recording, Annotation}\") # hide","category":"page"},{"location":"#Load-a-Dataset","page":"SpeechDatasets.jl","title":"Load a Dataset","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"dataset(name::AbstractString, inputdir::AbstractString, outputdir::AbstractString)\nBase.summary(dataset::SpeechDataset)\nget_dataset_kwargs(name::String)\n","category":"page"},{"location":"#SpeechDatasets.dataset-Tuple{AbstractString, AbstractString, AbstractString}","page":"SpeechDatasets.jl","title":"SpeechDatasets.dataset","text":"dataset(name::AbstractString, inputdir::AbstractString, outputdir::AbstractString; <keyword arguments>)\n\nExtract recordings and annotations for desired dataset.\n\nReturn a SpeechDataset object.\n\nCreate the outputdir folder, with:\n\nrecordings.jsonl containing each audio file path and associated metadata\nannotations-<subset>.jsonl containing each annotation and associated metadata\n\nArguments\n\nname Name of the dataset. Supported names are [\"AVID\", \"INA Diachrony\", \"Mini LibriSpeech\", \"Multilingual LibriSpeech\", \"TIMIT\", \"Speech2Tex\"].\ninputdir Name of dataset directory. If the directory does not exists, it is created and the data is downloaded if possible. Not all datasets can be downloaded, for example proprietary datasets does not implements a download function.\noutputdir is the output directory for manifest files.\n\nKeyword Arguments\n\nCommon kwargs are\n\nsubset Part of the dataset to load (for example \"train\" or \"test\").\nlang ISO 639-3 code of the language.\n\nOther kwargs can be available depending on the dataset, they can be accessed with get_dataset_kwargs(name::String).\n\n\n\n\n\n","category":"method"},{"location":"#Base.summary-Tuple{SpeechDataset}","page":"SpeechDatasets.jl","title":"Base.summary","text":"Base.summary(dataset::SpeechDataset)\n\nDisplay informations about given SpeechDataset\n\n\n\n\n\n","category":"method"},{"location":"#SpeechDatasets.get_dataset_kwargs-Tuple{String}","page":"SpeechDatasets.jl","title":"SpeechDatasets.get_dataset_kwargs","text":"get_dataset_kwargs(name::String)\n\nReturn a NamedTuple containing each supported kwarg and its default value for a dataset identified by name.\n\n\n\n\n\n","category":"method"},{"location":"#Types","page":"SpeechDatasets.jl","title":"Types","text":"","category":"section"},{"location":"#SpeechDataset","page":"SpeechDatasets.jl","title":"SpeechDataset","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"SpeechDatasetInfos\nSpeechDatasetInfos(name::AbstractString)\nSpeechDataset\nSpeechDataset(infos::SpeechDatasetInfos, manifestroot::AbstractString, subset::AbstractString)","category":"page"},{"location":"#SpeechDatasets.SpeechDatasetInfos","page":"SpeechDatasets.jl","title":"SpeechDatasets.SpeechDatasetInfos","text":"struct SpeechDatasetInfos\n\nStore metadata about a dataset.\n\nFields\n\nname Dataset official name\nlang Language or list of languages (ISO 639-3 code)\nlicense License name\nsource URL to the dataset publication or content\nauthors list of authors\ndescription A few sentences describing the content or main purpose \nsubsets List of available subsets (for example [\"train\", \"test\"])\n\n\n\n\n\n","category":"type"},{"location":"#SpeechDatasets.SpeechDatasetInfos-Tuple{AbstractString}","page":"SpeechDatasets.jl","title":"SpeechDatasets.SpeechDatasetInfos","text":"SpeechDatasetInfos(name::AbstractString)\n\nConstruct a SpeechDatasetInfos from the Dataset name.\n\n\n\n\n\n","category":"method"},{"location":"#SpeechDatasets.SpeechDataset","page":"SpeechDatasets.jl","title":"SpeechDatasets.SpeechDataset","text":"struct SpeechDataset <: MLUtils.AbstractDataContainer\n\nStore all dataset recordings and annotations. \n\nIt can be iterated, and will give a Tuple{Recording, Annotation} for each entry. Indexation can be done with integer or id.\n\nFields\n\ninfos::SpeechDatasetInfos\nidxs::Vector{AbstractString} id indexes to access elements\nannotations::Dict{AbstractString, Annotation} Annotation for each index\nrecordings::Dict{AbstractString, Recording} Recording for each index\n\n\n\n\n\n","category":"type"},{"location":"#SpeechDatasets.SpeechDataset-Tuple{SpeechDatasetInfos, AbstractString, AbstractString}","page":"SpeechDatasets.jl","title":"SpeechDatasets.SpeechDataset","text":"SpeechDataset(infos::SpeechDatasetInfos, manifestroot::AbstractString, subset::AbstractString)\n\nCreate a SpeechDataset from manifest files and subset.\n\n\n\n\n\n","category":"method"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"Access a single element with integer or id indexing","category":"page"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"# ds::SpeechDataset\nds[1]\nds[\"1988-147956-0027\"]","category":"page"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"Access several elements by providing a list","category":"page"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"ds[[1,4,7]]\nds[[8, 2, \"777-126732-0015\"]]","category":"page"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"Get all annotations","category":"page"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"ds.annotations","category":"page"},{"location":"#Manifest-items","page":"SpeechDatasets.jl","title":"Manifest items","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"SpeechDatasets.ManifestItem\nRecording\nAnnotation\nAudioSources.load(r::Recording; start = -1, duration = -1, channels = r.channels)\nAudioSources.load(r::Recording, a::Annotation)\nSpeechDatasets.load_manifest(T::Type{<:Union{Recording, Annotation}}, path)","category":"page"},{"location":"#SpeechDatasets.ManifestItem","page":"SpeechDatasets.jl","title":"SpeechDatasets.ManifestItem","text":"abstract type ManifestItem end\n\nBase class for all manifest item. Every manifest item should have an id attribute.\n\n\n\n\n\n","category":"type"},{"location":"#SpeechDatasets.Recording","page":"SpeechDatasets.jl","title":"SpeechDatasets.Recording","text":"struct Recording{Ts<:AbstractAudioSource} <: ManifestItem\n id::AbstractString\n source::Ts\n channels::Vector{Int}\n samplerate::Int\nend\n\nA recording is an audio source associated with and id.\n\nConstructors\n\nRecording(id, source, channels, samplerate)\nRecording(id, source[; channels = missing, samplerate = missing])\n\nIf the channels or the sample rate are not provided then they will be read from source.\n\nwarning: Warning\nWhen preparing large corpus, not providing the channels and/or the sample rate can drastically reduce the speed as it forces to read source.\n\n\n\n\n\n","category":"type"},{"location":"#SpeechDatasets.Annotation","page":"SpeechDatasets.jl","title":"SpeechDatasets.Annotation","text":"struct Annotation <: ManifestItem\n id::AbstractString\n recording_id::AbstractString\n start::Float64\n duration::Float64\n channel::Union{Vector, Colon}\n data::Dict\nend\n\nAn \"annotation\" defines a segment of a recording on a single channel. The data field is an arbitrary dictionary holdin the nature of the annotation. start and duration (in seconds) defines, where the segment is locatated within the recoding recording_id.\n\nConstructor\n\nAnnotation(id, recording_id, start, duration, channel, data)\nAnnotation(id, recording_id[; channel = missing, start = -1, duration = -1, data = missing)\n\nIf start and/or duration are negative, the segment is considered to be the whole sequence length of the recording.\n\n\n\n\n\n","category":"type"},{"location":"#AudioSources.load-Tuple{Recording}","page":"SpeechDatasets.jl","title":"AudioSources.load","text":"load(recording::Recording [; start = -1, duration = -1, channels = recording.channels])\nload(recording, annotation)\n\nLoad the signal from a recording. start, duration (in seconds)\n\nThe function returns a tuple (x, sr) where x is a NC array\n\nN is the length of the signal and C is the number of channels\nand sr is the sampling rate of the signal.\n\n\n\n\n\n","category":"method"},{"location":"#AudioSources.load-Tuple{Recording, Annotation}","page":"SpeechDatasets.jl","title":"AudioSources.load","text":"load(r::Recording, a::Annotation)\nload(t::Tuple{Recording, Annotation})\n\nLoad only a segment of the recording referenced in the annotation.\n\n\n\n\n\n","category":"method"},{"location":"#SpeechDatasets.load_manifest-Tuple{Type{<:Union{Annotation, Recording}}, Any}","page":"SpeechDatasets.jl","title":"SpeechDatasets.load_manifest","text":"load_manifest(Annotation, path)\nload_manifest(Recording, path)\n\nLoad Recording/Annotation manifest from path.\n\n\n\n\n\n","category":"method"},{"location":"#Lexicons","page":"SpeechDatasets.jl","title":"Lexicons","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"CMUDICT(path)\nTIMITDICT(timitdir)\nMFAFRDICT(path)","category":"page"},{"location":"#SpeechDatasets.CMUDICT-Tuple{Any}","page":"SpeechDatasets.jl","title":"SpeechDatasets.CMUDICT","text":"CMUDICT(path)\n\nReturn the dictionary of pronunciation loaded from the CMU sphinx dictionary. The CMU dictionary will be donwloaded and stored into to path. Subsequent calls will only read the file path without downloading again the data.\n\n\n\n\n\n","category":"method"},{"location":"#SpeechDatasets.TIMITDICT-Tuple{Any}","page":"SpeechDatasets.jl","title":"SpeechDatasets.TIMITDICT","text":"TIMITDICT(timitdir)\n\nReturn the dictionary of pronunciation as provided by TIMIT corpus (located in timitdir).\n\n\n\n\n\n","category":"method"},{"location":"#SpeechDatasets.MFAFRDICT-Tuple{Any}","page":"SpeechDatasets.jl","title":"SpeechDatasets.MFAFRDICT","text":"MFAFRDICT(path)\n\nReturn the french dictionary of pronunciation as provided by MFA (french_mfa v2.0.0a).\n\n\n\n\n\n","category":"method"},{"location":"#Index","page":"SpeechDatasets.jl","title":"Index","text":"","category":"section"},{"location":"","page":"SpeechDatasets.jl","title":"SpeechDatasets.jl","text":"","category":"page"},{"location":"datasets/#Supported-Datasets","page":"Supported Datasets","title":"Supported Datasets","text":"","category":"section"},{"location":"datasets/","page":"Supported Datasets","title":"Supported Datasets","text":"using SpeechDatasets, JSON, Markdown\ncorpora_infos = JSON.parsefile(SpeechDatasets.corpora_file)\n\nfunction write_corpora_docs(io::IO)\n for corpus in corpora_infos\n fields = keys(corpus)\n println(io, \"## $(corpus[\"name\"])\")\n\n if \"license\" in fields\n license = replace(corpus[\"license\"], \"-\" => \"--\") # dash are escaped\n color = license==\"proprietary\" ? \"red\" : \"lightblue\"\n license_badge = \"https://img.shields.io/badge/License-$license-$color\"\n println(io, \"![License]($license_badge)\")\n end\n\n if \"lang\" in fields\n languages = corpus[\"lang\"] isa String ? [corpus[\"lang\"]] : corpus[\"lang\"]\n for lang in languages\n lang_badge = \"https://img.shields.io/badge/Lang-$lang-lightgreen\"\n println(io, \"![Language]($lang_badge)\")\n end\n end\n\n if \"description\" in fields\n println(io, \"\")\n println(io, corpus[\"description\"])\n end\n\n if \"source\" in fields\n println(io, \"\")\n println(io, \"[Source]($(corpus[\"source\"]))\")\n end\n\n if \"authors\" in fields\n println(io, \"\")\n println(io, \"### Authors\")\n println(io, join(corpus[\"authors\"], \", \"))\n end\n\n if \"subsets\" in fields\n println(io, \"\")\n println(io, \"### Subsets\")\n println(io, join(corpus[\"subsets\"], \", \"))\n end\n\n kwargs = get_dataset_kwargs(corpus[\"name\"])\n if ! isempty(kwargs)\n println(io, \"### Keyword arguments\")\n println(io, \"```julia\")\n println(io, kwargs)\n println(io, \"```\")\n end\n\n println(io, \"\\n---\")\n end\nend\n\nMarkdown.parse(sprint(write_corpora_docs))","category":"page"}]
}
dev/siteinfo.js 0 → 100644
View file @ a28aaed1
var DOCUMENTER_CURRENT_VERSION = "dev";
index.html 0 → 100644
View file @ a28aaed1
<!--This file is automatically generated by Documenter.jl-->
<meta http-equiv="refresh" content="0; url=./dev/"/>
versions.js 0 → 100644
View file @ a28aaed1
var DOC_VERSIONS = [
"dev",
];
var DOCUMENTER_NEWEST = "dev";
var DOCUMENTER_STABLE = "dev";