add download by chunck

maap-s3.py

@@ -293,9 +293,9 @@ def download(path, name):
         if location:
             #We download the data using the location
             print("[INFO] we are about to download the data")
-            
-            response = requests.get(location)
-            open(name, 'wb').write(response.content)
+            download_file(location, name)
+            #response = requests.get(location)
+            #open(name, 'wb').write(response.content)
             print("[INFO] Download finished")
         
     else:
@@ -303,6 +303,21 @@ def download(path, name):
 
 
 
+##########################
+# download file using url #
+##########################       
+def download_file(url, name):
+    #local_filename = url.split('/')[-1]
+    # NOTE the stream=True parameter below
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(name, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192): 
+                # If you have chunk encoded response uncomment if
+                # and set chunk_size parameter to None.
+                #if chunk: 
+                f.write(chunk)
+    return name
 
 ##########################
 # list data in s3 folder #