├── .gitignore ├── LICENSE ├── README.md ├── docs ├── Makefile └── source │ ├── conf.py │ ├── index.rst │ ├── operators.rst │ ├── overview.rst │ ├── storage_driver.rst │ ├── task_runners.rst │ └── utils.rst ├── fileflow ├── __init__.py ├── configuration.py ├── errors.py ├── example_dags │ └── fileflow_example.py ├── operators │ ├── __init__.py │ ├── dive_operator.py │ └── dive_python_operator.py ├── storage_drivers │ ├── __init__.py │ ├── file_storage_driver.py │ ├── s3_storage_driver.py │ └── storage_driver.py ├── task_runners │ ├── __init__.py │ └── task_runner.py └── utils │ ├── __init__.py │ └── dataframe_utils.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── fixtures │ ├── FileStorageDriverReadTest.txt │ ├── SampleUniformData.json │ ├── example_dates │ │ ├── 2016-01-01 │ │ ├── 2016-01-02 │ │ └── 2016-01-03 │ └── utils │ │ ├── empty_dataframelike_csv.csv │ │ ├── example-dataframe.csv │ │ └── test_malformed_csv_ingest_tiny.csv ├── storage_drivers │ ├── __init__.py │ ├── test_fileStorageDriver.py │ ├── test_get_storage_driver.py │ └── test_s3StorageDriver.py ├── task_runners │ ├── __init__.py │ └── test_taskRunner.py └── utils │ └── test_pandas_io.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/README.md -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/source/operators.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/operators.rst -------------------------------------------------------------------------------- /docs/source/overview.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/overview.rst -------------------------------------------------------------------------------- /docs/source/storage_driver.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/storage_driver.rst -------------------------------------------------------------------------------- /docs/source/task_runners.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/task_runners.rst -------------------------------------------------------------------------------- /docs/source/utils.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/docs/source/utils.rst -------------------------------------------------------------------------------- /fileflow/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.0.3' 2 | -------------------------------------------------------------------------------- /fileflow/configuration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/configuration.py -------------------------------------------------------------------------------- /fileflow/errors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/errors.py -------------------------------------------------------------------------------- /fileflow/example_dags/fileflow_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/example_dags/fileflow_example.py -------------------------------------------------------------------------------- /fileflow/operators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/operators/__init__.py -------------------------------------------------------------------------------- /fileflow/operators/dive_operator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/operators/dive_operator.py -------------------------------------------------------------------------------- /fileflow/operators/dive_python_operator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/operators/dive_python_operator.py -------------------------------------------------------------------------------- /fileflow/storage_drivers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/storage_drivers/__init__.py -------------------------------------------------------------------------------- /fileflow/storage_drivers/file_storage_driver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/storage_drivers/file_storage_driver.py -------------------------------------------------------------------------------- /fileflow/storage_drivers/s3_storage_driver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/storage_drivers/s3_storage_driver.py -------------------------------------------------------------------------------- /fileflow/storage_drivers/storage_driver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/storage_drivers/storage_driver.py -------------------------------------------------------------------------------- /fileflow/task_runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/task_runners/__init__.py -------------------------------------------------------------------------------- /fileflow/task_runners/task_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/task_runners/task_runner.py -------------------------------------------------------------------------------- /fileflow/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/utils/__init__.py -------------------------------------------------------------------------------- /fileflow/utils/dataframe_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/fileflow/utils/dataframe_utils.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/setup.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/FileStorageDriverReadTest.txt: -------------------------------------------------------------------------------- 1 | this is a reading test. -------------------------------------------------------------------------------- /tests/fixtures/SampleUniformData.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/fixtures/SampleUniformData.json -------------------------------------------------------------------------------- /tests/fixtures/example_dates/2016-01-01: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/example_dates/2016-01-02: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/example_dates/2016-01-03: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/utils/empty_dataframelike_csv.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/fixtures/utils/empty_dataframelike_csv.csv -------------------------------------------------------------------------------- /tests/fixtures/utils/example-dataframe.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/fixtures/utils/example-dataframe.csv -------------------------------------------------------------------------------- /tests/fixtures/utils/test_malformed_csv_ingest_tiny.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/fixtures/utils/test_malformed_csv_ingest_tiny.csv -------------------------------------------------------------------------------- /tests/storage_drivers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/storage_drivers/test_fileStorageDriver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/storage_drivers/test_fileStorageDriver.py -------------------------------------------------------------------------------- /tests/storage_drivers/test_get_storage_driver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/storage_drivers/test_get_storage_driver.py -------------------------------------------------------------------------------- /tests/storage_drivers/test_s3StorageDriver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/storage_drivers/test_s3StorageDriver.py -------------------------------------------------------------------------------- /tests/task_runners/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/task_runners/test_taskRunner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/task_runners/test_taskRunner.py -------------------------------------------------------------------------------- /tests/utils/test_pandas_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tests/utils/test_pandas_io.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/industrydive/fileflow/HEAD/tox.ini --------------------------------------------------------------------------------