22
33__copyright__ = """
44
5- Copyright 2016 Lukasz Tracewski
5+ Copyright 2023 Samapriya Roy
66
77 Licensed under the Apache License, Version 2.0 (the "License");
88 you may not use this file except in compliance with the License.
1919"""
2020__license__ = "Apache 2.0"
2121
22- __Modifications_copyright__ = """
23-
24- Copyright 2022 Samapriya Roy
25-
26- Licensed under the Apache License, Version 2.0 (the "License");
27- you may not use this file except in compliance with the License.
28- You may obtain a copy of the License at
29-
30- http://www.apache.org/licenses/LICENSE-2.0
31-
32- Unless required by applicable law or agreed to in writing, software
33- distributed under the License is distributed on an "AS IS" BASIS,
34- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
35- See the License for the specific language governing permissions and
36- limitations under the License.
37-
38- """
39- __license__ = "Apache 2.0"
40-
41- """
42- Modifications to file:
43- - Uses cookie based upload
44- - Removed multipart upload
45- - Uses polling
46- """
47-
4822import ast
4923import csv
5024import glob
@@ -84,6 +58,19 @@ def _format_message(self, field, error):
8458 return "GEE file name & path cannot have spaces & can only have letters, numbers, hyphens and underscores"
8559
8660
61+ def task_counter ():
62+ ee .Initialize ()
63+ status = ["RUNNING" , "PENDING" ]
64+ task_count = len (
65+ [
66+ task
67+ for task in ee .data .listOperations ()
68+ if task ["metadata" ]["state" ] in status
69+ ]
70+ )
71+ return task_count
72+
73+
8774def upload (
8875 user ,
8976 source_path ,
@@ -133,19 +120,13 @@ def upload(
133120 # logging.info(
134121 # f"Processing image {current_image_no + 1} out of {no_images} : {image_path}"
135122 # )
136- status = ["RUNNING" , "PENDING" ]
137- task_count = len (
138- [
139- task
140- for task in ee .data .listOperations ()
141- if task ["metadata" ]["state" ] in status
142- ]
143- )
144- while task_count >= 2500 :
123+ task_count = task_counter ()
124+ while task_count >= 2800 :
145125 logging .info (
146126 f"Total tasks running or submitted { task_count } : waiting for 5 minutes"
147127 )
148128 time .sleep (300 )
129+ task_count = task_counter ()
149130 filename = __get_filename_from_path (path = image_path )
150131
151132 destination_path = ee .data .getAsset (destination_path + "/" )["name" ]
@@ -165,7 +146,7 @@ def upload(
165146
166147 df = pd .read_csv (metadata_path )
167148 dd = (df .applymap (type ) == str ).all (0 )
168- for ind , val in dd .iteritems ():
149+ for ind , val in dd .items ():
169150 if val == True :
170151 slist .append (ind )
171152 intcol = list (df .select_dtypes (include = ["int64" ]).columns )
0 commit comments