Gecko [ v3.cesa.co.za ]

Name	Size	Permission	Date
__pycache__	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
addlhelp	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
commands	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
data	[ DIR ]	drwxr-xr-x	2026-06-08 18:07
tests	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
third_party	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
utils	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
vendored	[ DIR ]	drwxr-xr-x	2026-06-08 18:08
README	378 B	-rw-r--r--	1980-01-01 08:00
__init__.py	6.3 KB	-rw-r--r--	1980-01-01 08:00
__main__.py	34.63 KB	-rw-r--r--	1980-01-01 08:00
boto_resumable_upload.py	24.69 KB	-rw-r--r--	1980-01-01 08:00
boto_translation.py	85.52 KB	-rw-r--r--	1980-01-01 08:00
bucket_listing_ref.py	3.9 KB	-rw-r--r--	1980-01-01 08:00
cloud_api.py	40.97 KB	-rw-r--r--	1980-01-01 08:00
cloud_api_delegator.py	29.87 KB	-rw-r--r--	1980-01-01 08:00
command.py	109.98 KB	-rw-r--r--	1980-01-01 08:00
command_argument.py	4.35 KB	-rw-r--r--	1980-01-01 08:00
command_runner.py	23.21 KB	-rw-r--r--	1980-01-01 08:00
context_config.py	8.36 KB	-rw-r--r--	1980-01-01 08:00
cred_types.py	1.15 KB	-rw-r--r--	1980-01-01 08:00
cs_api_map.py	3.31 KB	-rw-r--r--	1980-01-01 08:00
daisy_chain_wrapper.py	14.39 KB	-rw-r--r--	1980-01-01 08:00
devshell_auth_plugin.py	1.69 KB	-rw-r--r--	1980-01-01 08:00
discard_messages_queue.py	1.29 KB	-rw-r--r--	1980-01-01 08:00
exception.py	5.18 KB	-rw-r--r--	1980-01-01 08:00
file_part.py	3.39 KB	-rw-r--r--	1980-01-01 08:00
gcs_json_api.py	104.02 KB	-rw-r--r--	1980-01-01 08:00
gcs_json_credentials.py	20.31 KB	-rw-r--r--	1980-01-01 08:00
gcs_json_media.py	28.5 KB	-rw-r--r--	1980-01-01 08:00
help_provider.py	4.26 KB	-rw-r--r--	1980-01-01 08:00
iamcredentials_api.py	11.72 KB	-rw-r--r--	1980-01-01 08:00
impersonation_credentials.py	2.51 KB	-rw-r--r--	1980-01-01 08:00
kms_api.py	13.24 KB	-rw-r--r--	1980-01-01 08:00
lazy_wrapper.py	2.26 KB	-rw-r--r--	1980-01-01 08:00
metrics.py	38.98 KB	-rw-r--r--	1980-01-01 08:00
metrics_reporter.py	4.61 KB	-rw-r--r--	1980-01-01 08:00
metrics_tuple.py	1.44 KB	-rw-r--r--	1980-01-01 08:00
name_expansion.py	29.97 KB	-rw-r--r--	1980-01-01 08:00
no_op_auth_plugin.py	1.19 KB	-rw-r--r--	1980-01-01 08:00
no_op_credentials.py	1.03 KB	-rw-r--r--	1980-01-01 08:00
parallel_tracker_file.py	12.14 KB	-rw-r--r--	1980-01-01 08:00
plurality_checkable_iterator.py	4.4 KB	-rw-r--r--	1980-01-01 08:00
progress_callback.py	6.08 KB	-rw-r--r--	1980-01-01 08:00
project_id.py	1.69 KB	-rw-r--r--	1980-01-01 08:00
pubsub_api.py	11.17 KB	-rw-r--r--	1980-01-01 08:00
resumable_streaming_upload.py	8.53 KB	-rw-r--r--	1980-01-01 08:00
seek_ahead_thread.py	5.11 KB	-rw-r--r--	1980-01-01 08:00
sig_handling.py	5.64 KB	-rw-r--r--	1980-01-01 08:00
storage_uri_builder.py	2.02 KB	-rw-r--r--	1980-01-01 08:00
storage_url.py	16.16 KB	-rw-r--r--	1980-01-01 08:00
tab_complete.py	10.77 KB	-rw-r--r--	1980-01-01 08:00
thread_message.py	13.71 KB	-rw-r--r--	1980-01-01 08:00
tracker_file.py	24.23 KB	-rw-r--r--	1980-01-01 08:00
tz_utc.py	2.2 KB	-rw-r--r--	1980-01-01 08:00
ui_controller.py	47.42 KB	-rw-r--r--	1980-01-01 08:00
wildcard_iterator.py	37.73 KB	-rw-r--r--	1980-01-01 08:00

Rename

# -*- coding: utf-8 -*-
# Copyright 2014 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions for progress callbacks."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import time

from gslib.thread_message import ProgressMessage
from gslib.utils import parallelism_framework_util

# Default upper and lower bounds for progress callback frequency.
_START_BYTES_PER_CALLBACK = 1024 * 256
_MAX_BYTES_PER_CALLBACK = 1024 * 1024 * 100
_TIMEOUT_SECONDS = 1

# Max width of URL to display in progress indicator. Wide enough to allow
# 15 chars for x/y display on an 80 char wide terminal.
MAX_PROGRESS_INDICATOR_COLUMNS = 65

class ProgressCallbackWithTimeout(object):
  """Makes progress callbacks at least once every _TIMEOUT_SECONDS.

This prevents wrong throughput calculation while not being excessively
  overwhelming.
  """

def __init__(self,
               total_size,
               callback_func,
               start_bytes_per_callback=_START_BYTES_PER_CALLBACK,
               timeout=_TIMEOUT_SECONDS):
    """Initializes the callback with timeout.

Args:
      total_size: Total bytes to process. If this is None, size is not known
          at the outset.
      callback_func: Func of (int: processed_so_far, int: total_bytes)
          used to make callbacks.
      start_bytes_per_callback: Lower bound of bytes per callback.
      timeout: Number maximum of seconds without a callback.

"""
    self._bytes_per_callback = start_bytes_per_callback
    self._callback_func = callback_func
    self._total_size = total_size
    self._last_time = time.time()
    self._timeout = timeout
    self._bytes_processed_since_callback = 0
    self._callbacks_made = 0
    self._total_bytes_processed = 0

def Progress(self, bytes_processed):
    """Tracks byte processing progress, making a callback if necessary."""
    self._bytes_processed_since_callback += bytes_processed
    cur_time = time.time()
    if (self._bytes_processed_since_callback > self._bytes_per_callback or
        (self._total_size is not None and self._total_bytes_processed +
         self._bytes_processed_since_callback >= self._total_size) or
        (self._last_time - cur_time) > self._timeout):
      self._total_bytes_processed += self._bytes_processed_since_callback
      # TODO: We check if >= total_size and truncate because JSON uploads count
      # multipart metadata during their send progress. If the size is unknown,
      # we can't do this and the progress message will make it appear that we
      # send more than the original stream.
      if self._total_size is not None:
        bytes_sent = min(self._total_bytes_processed, self._total_size)
      else:
        bytes_sent = self._total_bytes_processed
      self._callback_func(bytes_sent, self._total_size)
      self._bytes_processed_since_callback = 0
      self._callbacks_made += 1
      self._last_time = cur_time

class FileProgressCallbackHandler(object):
  """Tracks progress info for large operations like file copy or hash.

Information is sent to the status_queue, which will print it in the
      UI Thread.
  """

def __init__(self,
               status_queue,
               start_byte=0,
               override_total_size=None,
               src_url=None,
               component_num=None,
               dst_url=None,
               operation_name=None):
    """Initializes the callback handler.

Args:
      status_queue: Queue for posting status messages for UI display.
      start_byte: The beginning of the file component, if one is being used.
      override_total_size: The size of the file component, if one is being used.
      src_url: FileUrl/CloudUrl representing the source file.
      component_num: Indicates the component number, if any.
      dst_url: FileUrl/CloudUrl representing the destination file, or None
        for unary operations like hashing.
      operation_name: String representing the operation name
    """
    self._status_queue = status_queue
    self._start_byte = start_byte
    self._override_total_size = override_total_size
    self._component_num = component_num
    self._src_url = src_url
    self._dst_url = dst_url
    self._operation_name = operation_name
    # Ensures final newline is written once even if we get multiple callbacks.
    self._last_byte_written = False

# Function signature is in boto callback format, which cannot be changed.
  def call(
      self,  # pylint: disable=invalid-name
      last_byte_processed,
      total_size):
    """Gathers information describing the operation progress.

Actual message is printed to stderr by UIThread.

Args:
      last_byte_processed: The last byte processed in the file. For file
                           components, this number should be in the range
                           [start_byte:start_byte + override_total_size].
      total_size: Total size of the ongoing operation.
    """
    if self._last_byte_written:
      return

if self._override_total_size:
      total_size = self._override_total_size

parallelism_framework_util.PutToQueueWithTimeout(
        self._status_queue,
        ProgressMessage(total_size,
                        last_byte_processed - self._start_byte,
                        self._src_url,
                        time.time(),
                        component_num=self._component_num,
                        operation_name=self._operation_name,
                        dst_url=self._dst_url))
    if total_size and last_byte_processed - self._start_byte == total_size:
      self._last_byte_written = True