镜像站点 · 本页由第三方 GitHub 只读镜像提供,非 GitHub 官方站点,不接受任何登录或凭据输入。前往 github.com
Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions bigquery/api/load_data_by_post.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,26 +50,35 @@ def load_data(schema_path, data_path, project_id, dataset_id, table_id):
credentials = GoogleCredentials.get_application_default()
bigquery = discovery.build('bigquery', 'v2', credentials=credentials)

# Infer the data format from the name of the data file.
source_format = 'CSV'
if data_path[-5:].lower() == '.json':
source_format = 'NEWLINE_DELIMITED_JSON'

# Post to the jobs resource using the client's media upload interface. See:
# http://developers.google.com/api-client-library/python/guide/media_upload
insert_request = bigquery.jobs().insert(
projectId=project_id,
# Provide a configuration object. See:
# https://cloud.google.com/bigquery/docs/reference/v2/jobs#resource
body={
"configuration": {
"load": {
"schema": {
"fields": json.load(open(schema_path, 'r'))
'configuration': {
'load': {
'schema': {
'fields': json.load(open(schema_path, 'r'))
},
'destinationTable': {
'projectId': project_id,
'datasetId': dataset_id,
'tableId': table_id
},
"destinationTable": {
"projectId": project_id,
"datasetId": dataset_id,
"tableId": table_id
}
'sourceFormat': source_format,
}
}
},
media_body=MediaFileUpload(
data_path,
mimetype="application/octet-stream"))

mimetype='application/octet-stream'))
job = insert_request.execute()

print('Waiting for job to finish...')
Expand All @@ -78,12 +87,14 @@ def load_data(schema_path, data_path, project_id, dataset_id, table_id):
projectId=job['jobReference']['projectId'],
jobId=job['jobReference']['jobId'])

# Poll the job until it finishes.
while True:
result = status_request.execute(num_retries=2)

if result['status']['state'] == 'DONE':
if 'errorResult' in result['status']:
raise RuntimeError(result['status']['errorResult'])
if result['status'].get('errors'):
raise RuntimeError('\n'.join(
e['message'] for e in result['status']['errors']))
print('Job complete.')
return

Expand Down