manyi/venv/Lib/site-packages/pandas/tests/io/conftest.py

from distutils.version import LooseVersion
import os

import pytest

import pandas.util.testing as tm

from pandas.io.parsers import read_csv


@pytest.fixture
def tips_file(datapath):
    """Path to the tips dataset"""
    return datapath('io', 'parser', 'data', 'tips.csv')


@pytest.fixture
def jsonl_file(datapath):
    """Path a JSONL dataset"""
    return datapath('io', 'parser', 'data', 'items.jsonl')


@pytest.fixture
def salaries_table(datapath):
    """DataFrame with the salaries dataset"""
    return read_csv(datapath('io', 'parser', 'data', 'salaries.csv'), sep='\t')


@pytest.fixture
def s3_resource(tips_file, jsonl_file):
    """Fixture for mocking S3 interaction.

    The primary bucket name is "pandas-test". The following datasets
    are loaded.

    - tips.csv
    - tips.csv.gz
    - tips.csv.bz2
    - items.jsonl

    A private bucket "cant_get_it" is also created. The boto3 s3 resource
    is yielded by the fixture.
    """
    pytest.importorskip('s3fs')
    boto3 = pytest.importorskip('boto3')
    botocore = pytest.importorskip('botocore')

    if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"):
        # botocore leaks an uncatchable ResourceWarning before 1.11.0;
        # see GH 23731 and https://github.com/boto/botocore/issues/1464
        pytest.skip("botocore is leaking resources before 1.11.0")

    with tm.ensure_safe_environment_variables():
        # temporary workaround as moto fails for botocore >= 1.11 otherwise,
        # see https://github.com/spulec/moto/issues/1924 & 1952
        os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
        os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")

        moto = pytest.importorskip('moto')

        test_s3_files = [
            ('tips.csv', tips_file),
            ('tips.csv.gz', tips_file + '.gz'),
            ('tips.csv.bz2', tips_file + '.bz2'),
            ('items.jsonl', jsonl_file),
        ]

        def add_tips_files(bucket_name):
            for s3_key, file_name in test_s3_files:
                with open(file_name, 'rb') as f:
                    conn.Bucket(bucket_name).put_object(
                        Key=s3_key,
                        Body=f)

        try:
            s3 = moto.mock_s3()
            s3.start()

            # see gh-16135
            bucket = 'pandas-test'
            conn = boto3.resource("s3", region_name="us-east-1")

            conn.create_bucket(Bucket=bucket)
            add_tips_files(bucket)

            conn.create_bucket(Bucket='cant_get_it', ACL='private')
            add_tips_files('cant_get_it')
            yield conn
        finally:
            s3.stop()