diff options
author | noptuno <repollo.marrero@gmail.com> | 2023-04-28 02:40:47 +0200 |
---|---|---|
committer | noptuno <repollo.marrero@gmail.com> | 2023-04-28 02:40:47 +0200 |
commit | 6f6a73987201c9c303047c61389b82ad98b15597 (patch) | |
tree | bf67eb590d49979d6740bc1e94b4018df48bce98 /venv/lib/python3.9/site-packages/smmap/buf.py | |
parent | Resolved merge conflicts and merged pr_218 into STREAMLIT_CHAT_IMPLEMENTATION (diff) | |
parent | Merging PR_218 openai_rev package with new streamlit chat app (diff) | |
download | gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.tar gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.tar.gz gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.tar.bz2 gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.tar.lz gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.tar.xz gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.tar.zst gpt4free-6f6a73987201c9c303047c61389b82ad98b15597.zip |
Diffstat (limited to 'venv/lib/python3.9/site-packages/smmap/buf.py')
-rw-r--r-- | venv/lib/python3.9/site-packages/smmap/buf.py | 143 |
1 files changed, 143 insertions, 0 deletions
diff --git a/venv/lib/python3.9/site-packages/smmap/buf.py b/venv/lib/python3.9/site-packages/smmap/buf.py new file mode 100644 index 00000000..795e0fd8 --- /dev/null +++ b/venv/lib/python3.9/site-packages/smmap/buf.py @@ -0,0 +1,143 @@ +"""Module with a simple buffer implementation using the memory manager""" +import sys + +__all__ = ["SlidingWindowMapBuffer"] + + +class SlidingWindowMapBuffer: + + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by the provided cursor. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at the offset you used during initialization or begin_access + + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) + + def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset + + def __del__(self): + self.end_access() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end_access() + + def __len__(self): + return self._size + + def __getitem__(self, i): + if isinstance(i, slice): + return self.__getslice__(i.start or 0, i.stop or self._size) + c = self._c + assert c.is_valid() + if i < 0: + i = self._size + i + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + return c.buffer()[i - c.ofs_begin()] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxsize: + j = self._size + if j < 0: + j = self._size + j + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i - b:j - b] + else: + l = j - i # total length + ofs = i + # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower + # in the previous iteration of this code + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() + md.append(d) + # END while there are bytes to read + return bytes().join(md) + # END fast or slow path + #{ Interface + + def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if cursor is None the existing one will be used. + :return: True if the buffer can be used""" + if cursor: + self._c = cursor + # END update our cursor + + # reuse existing cursors if possible + if self._c is not None and self._c.is_associated(): + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + # END handle size + self._size = size + # END set size + return res + # END use our cursor + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 + if self._c is not None: + self._c.unuse_region() + # END unuse region + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface |