Viewing File: /home/ubuntu/.local/lib/python3.10/site-packages/fastai/datasets.py
from .core import *
import hashlib
__all__ = ['URLs', 'Config', 'untar_data', 'download_data', 'datapath4file', 'url2name', 'url2path']
MODEL_URL = 'http://files.fast.ai/models/'
URL = 'http://files.fast.ai/data/examples/'
class URLs():
"Global constants for dataset and model URLs."
LOCAL_PATH = Path.cwd()
S3 = 'https://s3.amazonaws.com/fast-ai-'
S3_IMAGE = f'{S3}imageclas/'
S3_IMAGELOC = f'{S3}imagelocal/'
S3_NLP = f'{S3}nlp/'
S3_COCO = f'{S3}coco/'
S3_MODEL = f'{S3}modelzoo/'
# main datasets
ADULT_SAMPLE = f'{URL}adult_sample'
BIWI_SAMPLE = f'{URL}biwi_sample'
CIFAR = f'{URL}cifar10'
COCO_SAMPLE = f'{S3_COCO}coco_sample'
COCO_TINY = f'{URL}coco_tiny'
HUMAN_NUMBERS = f'{URL}human_numbers'
IMDB = f'{S3_NLP}imdb'
IMDB_SAMPLE = f'{URL}imdb_sample'
ML_SAMPLE = f'{URL}movie_lens_sample'
MNIST_SAMPLE = f'{URL}mnist_sample'
MNIST_TINY = f'{URL}mnist_tiny'
MNIST_VAR_SIZE_TINY = f'{S3_IMAGE}mnist_var_size_tiny'
PLANET_SAMPLE = f'{URL}planet_sample'
PLANET_TINY = f'{URL}planet_tiny'
IMAGENETTE = f'{S3_IMAGE}imagenette'
IMAGENETTE_160 = f'{S3_IMAGE}imagenette-160'
IMAGENETTE_320 = f'{S3_IMAGE}imagenette-320'
IMAGEWOOF = f'{S3_IMAGE}imagewoof'
IMAGEWOOF_160 = f'{S3_IMAGE}imagewoof-160'
IMAGEWOOF_320 = f'{S3_IMAGE}imagewoof-320'
# kaggle competitions download dogs-vs-cats -p {DOGS.absolute()}
DOGS = f'{URL}dogscats'
# image classification datasets
CALTECH_101 = f'{S3_IMAGE}caltech_101'
CARS = f'{S3_IMAGE}stanford-cars'
CIFAR_100 = f'{S3_IMAGE}cifar100'
CUB_200_2011 = f'{S3_IMAGE}CUB_200_2011'
FLOWERS = f'{S3_IMAGE}oxford-102-flowers'
FOOD = f'{S3_IMAGE}food-101'
MNIST = f'{S3_IMAGE}mnist_png'
PETS = f'{S3_IMAGE}oxford-iiit-pet'
# NLP datasets
AG_NEWS = f'{S3_NLP}ag_news_csv'
AMAZON_REVIEWS = f'{S3_NLP}amazon_review_full_csv'
AMAZON_REVIEWS_POLARITY = f'{S3_NLP}amazon_review_polarity_csv'
DBPEDIA = f'{S3_NLP}dbpedia_csv'
MT_ENG_FRA = f'{S3_NLP}giga-fren'
SOGOU_NEWS = f'{S3_NLP}sogou_news_csv'
WIKITEXT = f'{S3_NLP}wikitext-103'
WIKITEXT_TINY = f'{S3_NLP}wikitext-2'
YAHOO_ANSWERS = f'{S3_NLP}yahoo_answers_csv'
YELP_REVIEWS = f'{S3_NLP}yelp_review_full_csv'
YELP_REVIEWS_POLARITY = f'{S3_NLP}yelp_review_polarity_csv'
# Image localization datasets
BIWI_HEAD_POSE = f"{S3_IMAGELOC}biwi_head_pose"
CAMVID = f'{S3_IMAGELOC}camvid'
CAMVID_TINY = f'{URL}camvid_tiny'
LSUN_BEDROOMS = f'{S3_IMAGE}bedroom'
PASCAL_2007 = f'{S3_IMAGELOC}pascal_2007'
PASCAL_2012 = f'{S3_IMAGELOC}pascal_2012'
#Pretrained models
OPENAI_TRANSFORMER = f'{S3_MODEL}transformer'
WT103 = f'{S3_MODEL}wt103'
WT103_1 = f'{S3_MODEL}wt103-1'
# to create/update a checksum for ./mnist_var_size_tiny.tgz, run:
# python -c 'import fastai.datasets; print(fastai.datasets._check_file("mnist_var_size_tiny.tgz"))'
_checks = {
URLs.ADULT_SAMPLE:(968212, '64eb9d7e23732de0b138f7372d15492f'),
URLs.AG_NEWS:(11784419, 'b86f328f4dbd072486591cb7a5644dcd'),
URLs.AMAZON_REVIEWS_POLARITY:(688339454, '676f7e5208ec343c8274b4bb085bc938'),
URLs.AMAZON_REVIEWS:(643695014, '4a1196cf0adaea22f4bc3f592cddde90'),
URLs.BIWI_HEAD_POSE:(452316199, '00f4ccf66e8cba184bc292fdc08fb237'),
URLs.BIWI_SAMPLE:(593774, '9179f4c1435f4b291f0d5b072d60c2c9'),
URLs.CALTECH_101:(131740031, 'd673425306e98ee4619fcdeef8a0e876'),
URLs.CAMVID:(598913237, '648371e4f3a833682afb39b08a3ce2aa'),
URLs.CAMVID_TINY:(2314212, '2cf6daf91b7a2083ecfa3e9968e9d915'),
URLs.CARS:(1957803273, '9045d6673c9ced0889f41816f6bf2f9f'),
URLs.CIFAR:(168168549, 'a5f8c31371b63a406b23368042812d3c'),
URLs.CIFAR_100:(169168619, 'e5e65dcb54b9d3913f7b8a9ad6607e62'),
URLs.COCO_SAMPLE:(3245877008, '006cd55d633d94b36ecaf661467830ec'),
URLs.COCO_TINY:(801038, '367467451ac4fba79a647753c2c66d3a'),
URLs.CUB_200_2011:(1150585339, 'd2acaa99439dff0483c7bbac1bfe2a92'),
URLs.DBPEDIA:(68341743, '239c7837b9e79db34486f3de6a00e38e'),
URLs.DOGS:(839285364, '3e483c8d6ef2175e9d395a6027eb92b7'),
URLs.FLOWERS:(345236087, '5666e01c1311b4c67fcf20d2b3850a88'),
URLs.FOOD:(5686607260, '1a540ebf1fb40b2bf3f2294234ba7907'),
URLs.HUMAN_NUMBERS:(30252, '8a19c3bfa2bcb08cd787e741261f3ea2'),
URLs.IMDB:(144440600, '90f9b1c4ff43a90d67553c9240dc0249'),
URLs.IMDB_SAMPLE:(571827, '0842e61a9867caa2e6fbdb14fa703d61'),
URLs.LSUN_BEDROOMS:(4579163978, '35d84f38f8a15fe47e66e460c8800d68'),
URLs.ML_SAMPLE:(51790, '10961384dfe7c5181460390a460c1f77'),
URLs.MNIST:(15683414, '03639f83c4e3d19e0a3a53a8a997c487'),
URLs.MNIST_SAMPLE:(3214948, '2dbc7ec6f9259b583af0072c55816a88'),
URLs.MNIST_TINY:(342207, '56143e8f24db90d925d82a5a74141875'),
URLs.MNIST_VAR_SIZE_TINY:(565372, 'b71a930f4eb744a4a143a6c7ff7ed67f'),
URLs.MT_ENG_FRA:(2598183296, '69573f58e2c850b90f2f954077041d8c'),
URLs.OPENAI_TRANSFORMER:(432848315, '024b0d2203ebb0cd1fc64b27cf8af18e'),
URLs.PASCAL_2007:(1636130334, 'a70574e9bc592bd3b253f5bf46ce12e3'),
URLs.PASCAL_2012:(2611715776, '2ae7897038383836f86ce58f66b09e31'),
URLs.PETS:(811706944, 'e4db5c768afd933bb91f5f594d7417a4'),
URLs.PLANET_SAMPLE:(15523994, '8bfb174b3162f07fbde09b54555bdb00'),
URLs.PLANET_TINY:(997569, '490873c5683454d4b2611fb1f00a68a9'),
URLs.SOGOU_NEWS:(384269937, '950f1366d33be52f5b944f8a8b680902'),
URLs.WIKITEXT:(190200704, '2dd8cf8693b3d27e9c8f0a7df054b2c7'),
URLs.WIKITEXT_TINY:(4070055, '2a82d47a7b85c8b6a8e068dc4c1d37e7'),
URLs.WT103:(206789489, '76fd08236c78bf91b7fb76698d53afa3'),
URLs.WT103_1:(165175630, '9cbe02e9e23b969fee10dc9b8dec6566'),
URLs.YAHOO_ANSWERS:(319476345, '0632a0d236ef3a529c0fa4429b339f68'),
URLs.YELP_REVIEWS_POLARITY:(166373201, '48c8451c1ad30472334d856b5d294807'),
URLs.YELP_REVIEWS:(196146755, '1efd84215ea3e30d90e4c33764b889db'),
}
#TODO: This can probably be coded more shortly and nicely.
class Config():
"Creates a default config file 'config.yml' in $FASTAI_HOME (default `~/.fastai/`)"
DEFAULT_CONFIG_LOCATION = os.path.expanduser(os.getenv('FASTAI_HOME', '~/.fastai'))
DEFAULT_CONFIG_PATH = DEFAULT_CONFIG_LOCATION + '/config.yml'
DEFAULT_CONFIG = {
'data_path': DEFAULT_CONFIG_LOCATION + '/data',
'data_archive_path': DEFAULT_CONFIG_LOCATION + '/data',
'model_path': DEFAULT_CONFIG_LOCATION + '/models'
}
@classmethod
def get_key(cls, key):
"Get the path to `key` in the config file."
return cls.get().get(key, cls.DEFAULT_CONFIG.get(key,None))
@classmethod
def get_path(cls, path):
"Get the `path` in the config file."
return _expand_path(cls.get_key(path))
@classmethod
def data_path(cls):
"Get the path to data in the config file."
return cls.get_path('data_path')
@classmethod
def data_archive_path(cls):
"Get the path to data archives in the config file."
return cls.get_path('data_archive_path')
@classmethod
def model_path(cls):
"Get the path to fastai pretrained models in the config file."
return cls.get_path('model_path')
@classmethod
def get(cls, fpath=None, create_missing=True):
"Retrieve the `Config` in `fpath`."
fpath = _expand_path(fpath or cls.DEFAULT_CONFIG_PATH)
if not fpath.exists() and create_missing: cls.create(fpath)
assert fpath.exists(), f'Could not find config at: {fpath}. Please create'
with open(fpath, 'r') as yaml_file: return yaml.safe_load(yaml_file)
@classmethod
def create(cls, fpath):
"Creates a `Config` from `fpath`."
fpath = _expand_path(fpath)
assert(fpath.suffix == '.yml')
if fpath.exists(): return
fpath.parent.mkdir(parents=True, exist_ok=True)
with open(fpath, 'w') as yaml_file:
yaml.dump(cls.DEFAULT_CONFIG, yaml_file, default_flow_style=False)
def _expand_path(fpath): return Path(fpath).expanduser()
def url2name(url): return url.split('/')[-1]
#TODO: simplify this mess
def url2path(url, data=True, ext:str='.tgz'):
"Change `url` to a path."
name = url2name(url)
return datapath4file(name, ext=ext) if data else modelpath4file(name, ext=ext)
def _url2tgz(url, data=True, ext:str='.tgz'):
return datapath4file(f'{url2name(url)}{ext}', ext=ext) if data else modelpath4file(f'{url2name(url)}{ext}', ext=ext)
def modelpath4file(filename, ext:str='.tgz'):
"Return model path to `filename`, checking locally first then in the config file."
local_path = URLs.LOCAL_PATH/'models'/filename
if local_path.exists() or local_path.with_suffix(ext).exists(): return local_path
else: return Config.model_path()/filename
def datapath4file(filename, ext:str='.tgz', archive=True):
"Return data path to `filename`, checking locally first then in the config file."
local_path = URLs.LOCAL_PATH/'data'/filename
if local_path.exists() or local_path.with_suffix(ext).exists(): return local_path
elif archive: return Config.data_archive_path() / filename
else: return Config.data_path() / filename
def download_data(url:str, fname:PathOrStr=None, data:bool=True, ext:str='.tgz') -> Path:
"Download `url` to destination `fname`."
fname = Path(ifnone(fname, _url2tgz(url, data, ext=ext)))
os.makedirs(fname.parent, exist_ok=True)
if not fname.exists():
print(f'Downloading {url}')
download_url(f'{url}{ext}', fname)
return fname
def _check_file(fname):
size = os.path.getsize(fname)
with open(fname, "rb") as f:
hash_nb = hashlib.md5(f.read(2**20)).hexdigest()
return size,hash_nb
def untar_data(url:str, fname:PathOrStr=None, dest:PathOrStr=None, data=True, force_download=False) -> Path:
"Download `url` to `fname` if it doesn't exist, and un-tgz to folder `dest`."
dest = url2path(url, data) if dest is None else Path(dest)/url2name(url)
fname = Path(ifnone(fname, _url2tgz(url, data)))
if force_download or (fname.exists() and url in _checks and _check_file(fname) != _checks[url]):
print(f"A new version of the {'dataset' if data else 'model'} is available.")
if fname.exists(): os.remove(fname)
if dest.exists(): shutil.rmtree(dest)
if not dest.exists():
fname = download_data(url, fname=fname, data=data)
data_archive_dir = Config().data_archive_path()
if url in _checks:
assert _check_file(fname) == _checks[url], f"Downloaded file {fname} does not match checksum expected! Remove that file from {data_archive_dir} and try your code again."
tarfile.open(fname, 'r:gz').extractall(dest.parent)
return dest
Back to Directory
File Manager