Skip to content
Snippets Groups Projects
Commit b3de79d2 authored by kosted's avatar kosted
Browse files

stable version of the script

parent 62639cdd
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 32 additions and 27 deletions
maap-s3.py
+ 32
− 27
View file @ b3de79d2
......@@ -4,6 +4,7 @@ import json
import sys
import os
import time
import math
import os.path as path
# Import getopt module
import getopt
......@@ -19,7 +20,7 @@ def display_help():
print('Usage: [option...] {init|refresh|upload|download|ls|delete}')
print('init Get a fresh token before any request. It ask for email and password')
print('upload myFile.tiff locally path/myFile.tiff in the S3 Upload data in the S3')
print('download path/in/S3/file.tiff myFile.tiff Download a data from the S3')
print('download path/in/S3/file.tiff myFileName.tiff Download a data from the S3')
print('ls folder/path List data in a subfolder')
print('delete path/in/S3/file.tiff Delete an existing data on S3')
print('refresh refresh credentials and password')
......@@ -120,10 +121,7 @@ def upload(sourceFile, destination):
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
# If the file is less that 100 MB we upload directly
#Check file size
......@@ -136,7 +134,12 @@ def upload(sourceFile, destination):
print("[INFO] Starting multi part upload")
upload_multipart(sourceFile, destination)
else:
else:
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
print("[INFO] Starting retrieving the presigned url for the creation of the file with token "+ token)
#files = {'upload_file': open(sourceFile,'rb')}
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
......@@ -165,7 +168,13 @@ def upload(sourceFile, destination):
# Upload the data in S3, the data is split chunk by chunk #
###########################################################
def upload_multipart(sourceFile, destination):
#Get the token
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Set variables
filePath = sourceFile
key = destination
......@@ -175,21 +184,21 @@ def upload_multipart(sourceFile, destination):
#Set max to split to 5M
max_size = 5 * 1024 * 1024 # Approach 1: Assign the size
nbParts = math.ceil(fileSize/max_size) #calculate nbParts
print("[INFO] We will have "+nbParts+" parts")
print("[INFO] We will have "+ str(nbParts)+" parts")
url = "http://localhost:8080/s3/generateUploadId"
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateUploadId"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key}
response = requests.get(url, params = params)
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
print("[INFO] uploadId "+ response.text)
#Save upload id
uploadId = response.text
#Generate presigned urls
url = "http://localhost:8080/s3/generateListPresignedUrls"
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateListPresignedUrls"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, params = params)
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
stringList = response.text
str1 = stringList.replace(']','').replace('[','')
......@@ -218,9 +227,9 @@ def upload_multipart(sourceFile, destination):
#sys.stdout.close()
#complete the multi part
url = "http://localhost:8080/s3/completeMultiPartUploadRequest"
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/completeMultiPartUploadRequest"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, data=str(parts), params = params)
response = requests.get(url, data=str(parts), params = params, headers = {'Authorization': 'Bearer '+token})
###################
......@@ -261,7 +270,7 @@ def delete(destination):
###################
# download the data #
####################
def download(path):
def download(path, name):
print("[INFO] path file is : ", path)
if path:
......@@ -285,7 +294,7 @@ def download(path):
print("[INFO] we are about to download the data")
response = requests.get(location)
#open('google.ico', 'wb').write(r.content)
open(name, 'wb').write(response.content)
print("[INFO] Download finished")
else:
......@@ -338,34 +347,30 @@ try:
for opt, arg in options:
# Calculate the sum if the option is -a or --add
if opt in ('-u', '--upload'):
# Check the parameters
# Upload a data
if len(argv) != 3:
display_help()
else:
upload(argv[1], argv[2])
# Delete the data
elif opt in ('-r', '--remove'):
# Delete a data
if len(argv) != 2:
display_help()
else:
delete(argv[1])
elif opt in ('-l', '--list'):
# list a folder
if len(argv) != 2:
display_help()
else:
list(argv[1])
elif opt in ('-d', '--download'):
if len(argv) != 2:
# Download a data
if len(argv) != 3:
display_help()
else:
download(argv[1])
# Calculate the suntraction if the option is -s or --sub
elif opt in ('-s', '--sub'):
result = int(argv[1]) - int(argv[2])
print('Result = ', result)
download(argv[1], argv[2])
elif opt in ('-h', '--help'):
# Print the option
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment