Skip to content
Snippets Groups Projects
Commit 9fd082e2 authored by Teddy Kossoko's avatar Teddy Kossoko
Browse files

New version of the script with human reading name of the functions

parent ac326237
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitignore
+ 2
− 1
View file @ 9fd082e2
secrets/
maap-s3-userinfo.json
\ No newline at end of file
maap-s3-userinfo.json
maap-s3-multipartinfo.json
\ No newline at end of file
maap-s3.py
+ 162
− 48
View file @ 9fd082e2
......@@ -17,19 +17,23 @@ BEARER=""
#if windows we take the current folder
if sys.platform == 'win32':
USER_INFO_FILE_PATH=os.getcwd()+"\maap-s3-userinfo.json"
USER_LAST_UPLOAD_INFO_FILE_PATH=os.getcwd()+"\maap-s3-multipartinfo.json"
else :
USER_INFO_FILE_PATH="/usr/bmap/maap-s3-userinfo.json"
USER_LAST_UPLOAD_INFO_FILE_PATH="/usr/bmap/maap-s3-multipartinfo.json"
userinfo = {}
multipartinfo = {}
def display_help():
print('Usage: [option...] {-f|-u|-d|-l|-r}')
print('Usage: [option...] {upload|download|list|delete|refresh|resume}')
#print('-i Get a fresh token before any request. It ask for email and password')
print('-u myFile.tiff locally path/myFile.tiff in the S3 Upload data in the S3')
print('-d myFileName.tiff path/in/S3/file.tiff Download a data from the S3')
print('-l folder/path List data in a subfolder')
print('-r path/in/S3/file.tiff Delete an existing data on S3')
print('-f refresh credentials and password')
print('upload myFile.tiff locally path/myFile.tiff in the S3 Upload data in the S3')
print('download myFileName.tiff path/in/S3/file.tiff Download a data from the S3')
print('list folder/path List data in a subfolder')
print('delete path/in/S3/file.tiff Delete an existing data on S3')
print('refresh Refresh credentials and password')
print('resume Resume last interrupted multipart upload')
sys.exit(2)
......@@ -231,6 +235,18 @@ def upload_multipart(sourceFile, destination):
parts.append({'eTag': etag, 'partNumber': int(i+1)})
print(parts)
i = i+1
#We save also the multipart
#So we can resume it if upload failed
multipartinfo = {
'uploadId': uploadId,
'partsUpploaded': parts,
'sourceFile': sourceFile,
'destination': destination
}
#add the json in the file
with open(USER_LAST_UPLOAD_INFO_FILE_PATH, 'w') as outfile:
json.dump(multipartinfo, outfile)
#sys.stdout.close()
......@@ -238,7 +254,109 @@ def upload_multipart(sourceFile, destination):
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/completeMultiPartUploadRequest"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, data=str(parts), params = params, headers = {'Authorization': 'Bearer '+token})
#delete the file of multipart info because upload was success
os.remove(USER_LAST_UPLOAD_INFO_FILE_PATH)
###################################
# Resume failed multi part upload #
###################################
def resume():
print("[INFO] Resume the last multipart upload")
print("[INFO] Check last multipart upload metadata")
if os.path.isfile(USER_LAST_UPLOAD_INFO_FILE_PATH):
#Generate or get a token
init()
print("[INFO] Previous multi part upload file found")
token=''
#Get the token
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Get the data in the json file
with open(USER_LAST_UPLOAD_INFO_FILE_PATH) as json_file:
multipartinfo = json.load(json_file)
#Get the info
uploadId=multipartinfo['uploadId']
destination=multipartinfo['destination']
sourceFile=multipartinfo['sourceFile']
partsUpploaded=multipartinfo['partsUpploaded']
#We get the presigned url
fileSize = os.stat(sourceFile).st_size
print("Size "+ str(fileSize))
#Set max to split to 5M
max_size = 5 * 1024 * 1024 # Approach 1: Assign the size
nbParts = math.ceil(fileSize/max_size) #calculate nbParts
finalPart = nbParts - len(partsUpploaded)
print("[INFO] We will have "+ str(nbParts)+" parts minus already uploaded parts. We have to push "+ str(finalPart) +" parts")
#Generate presigned urls
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateListPresignedUrls"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': destination, 'nbParts': finalPart, 'uploadId': uploadId}
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
stringList = response.text
str1 = stringList.replace(']','').replace('[','')
listPresignedUrl = str1.replace('"','').split(",")
#Get the json uploaded list
#we iterate over the data and repush
with open(sourceFile, 'rb') as f:
i = 0
presignedUrlIndex = 0
while i < nbParts:
file_data = f.read(max_size)
#We upload only if we have new nn uploaded part
if i > len(partsUpploaded):
print("Upload part "+ str(i))
headers={'Content-Length': str(max_size)}
#print(listPresignedUrl[i])
response = requests.put(listPresignedUrl[presignedUrlIndex], data=file_data, headers=headers)
#print(response.headers)
#print(response.text)
etag = response.headers['ETag']
partsUpploaded.append({'eTag': etag, 'partNumber': int(i+1)})
presignedUrlIndex = presignedUrlIndex + 1
#We save also the multipart
#So we can resume it if upload failed
multipartinfo = {
'uploadId': uploadId,
'partsUpploaded': partsUpploaded,
'sourceFile': sourceFile,
'destination': destination
}
#add the json in the file
with open(USER_LAST_UPLOAD_INFO_FILE_PATH, 'w') as outfile:
json.dump(multipartinfo, outfile)
#We increase the data
i = i+1
#complete the multi part
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/completeMultiPartUploadRequest"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, data=str(parts), params = params, headers = {'Authorization': 'Bearer '+token})
#delete the file of multipart info because upload was success
os.remove(USER_LAST_UPLOAD_INFO_FILE_PATH)
else:
print("[INFO] Please run upload. There are no upload to be resume")
###################
# Delete the data #
......@@ -257,9 +375,10 @@ def delete(destination):
#call the api to delete the data
#Get the presigned url to delete the data
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
response = requests.delete(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
url = "http://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
print(url)
response = requests.delete(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
location = response.headers['Location']
#We have the
......@@ -326,6 +445,7 @@ def download_file(url, name):
f.write(chunk)
return name
##########################
# list data in s3 folder #
##########################
......@@ -357,52 +477,46 @@ def list(path):
# Store argument variable omitting the script name
argv = sys.argv[1:]
# Initialize result variable
result=0
try:
# Define getopt short and long options
options, args = getopt.getopt(sys.argv[1:], 'u:r:l:d:i:f:h', ['upload=', 'remove=','list=','download=','init=','refresh=','help='])
# Read each option using for loop
for opt, arg in options:
# Calculate the sum if the option is -a or --add
if opt in ('-u', '--upload'):
# Upload a data
if len(argv) != 3:
display_help()
else:
upload(argv[1], argv[2])
# Delete the data
elif opt in ('-r', '--remove'):
# Delete a data
if len(argv) != 2:
display_help()
else:
delete(argv[1])
elif opt in ('-l', '--list'):
# list a folder
if len(argv) != 2:
if len(argv) == 0:
display_help()
else:
if argv[0] == 'resume':
resume()
elif argv[0] == 'refresh':
refresh()
elif argv[0] == 'upload':
# Upload a data
if len(argv) != 3:
display_help()
else:
upload(argv[1], argv[2])
elif argv[0] == 'delete':
# Delete a data
if len(argv) != 2:
display_help()
else:
delete(argv[1])
elif argv[0] == 'download':
# Download a data
if len(argv) != 3:
display_help()
else:
download(argv[1], argv[2])
elif argv[0] == 'list':
# list a folder
if len(argv) != 2:
display_help()
else:
list(argv[1])
elif argv[0] == 'help':
display_help()
else:
list(argv[1])
elif opt in ('-d', '--download'):
# Download a data
if len(argv) != 3:
else:
display_help()
else:
download(argv[1], argv[2])
elif opt in ('-i', '--init'):
# Download a data
init()
elif opt in ('-f', '--refresh'):
# Download a data
refresh()
elif opt in ('-h', '--help'):
# Print the option
display_help()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment