Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ exclude: |
(?x)(
^scripts/disBatch/|
^tests/Mini_N64_L32/|
\.asdf$
)'
\.asdf$|
\.svg$
)

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
Expand Down
10 changes: 5 additions & 5 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ New Features
- Add pack9 reader [#25]
- Add light cone catalog reading to ``CompaSOHaloCatalog`` [#11]

Enhancements
Enhancements
~~~~~~~~~~~~
- Sped up RNG for reseeding [#24]

Expand Down Expand Up @@ -112,8 +112,8 @@ Changes

Fixes
~~~~~
- Fixed issue where satellite galaxy halo ID was incorrect.
- Fixed issue where satellite galaxy halo ID was incorrect.

New Features
~~~~~~~~~~~~
- ``CompaSOHaloCatalog`` can read "cleaned" halo catalogs with ``cleaned=True`` (the default) [#6]
Expand All @@ -135,7 +135,7 @@ Deprecations
~~~~~~~~~~~~
- Passing a string to the ``load_subsamples`` argument of ``CompaSOHaloCatalog`` is deprecated;
use a dict instead, like: ``load_subsamples=dict(A=True, rv=True)``. [#6]

- ``cleaned_halos`` renamed to ``cleaned``

0.4.0 (2021-02-03)
Expand All @@ -162,7 +162,7 @@ Fixes
~~~~~
- Specify minimum Astropy version to avoid
``AttributeError: 'numpy.ndarray' object has no attribute 'info'``

0.2.0 (2020-07-08)
------------------

Expand Down
2 changes: 2 additions & 0 deletions abacusnbody/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
from .version import __version__

__all__ = ['__version__']
53 changes: 27 additions & 26 deletions abacusnbody/data/asdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,38 @@
handle Blosc compression.
'''

import time
import struct
import time

import numpy as np
import asdf
import blosc
from asdf.extension import Extension, Compressor
import numpy as np
from asdf.extension import Compressor, Extension


import asdf
def _monkey_patch(*args,**kwargs):
raise Exception("Please use abacusnbody.data.asdf.set_nthreads(nthreads)")

asdf.compression.set_decompression_options = _monkey_patch

def set_nthreads(nthreads):
blosc.set_nthreads(nthreads)


class BloscCompressor(Compressor):
'''
An implementation of Blosc compression, as used by Abacus.
'''

@property
def label(self):
'''
The string labels in the binary block headers
that indicate Blosc compression
'''
return b'blsc'


def compress(self, data, **kwargs):
'''Useful compression kwargs:
nthreads
Expand All @@ -51,14 +52,14 @@ def compress(self, data, **kwargs):
'''
# Blosc code probably assumes contiguous buffer
assert data.contiguous

nthreads = kwargs.pop('nthreads', 1)
compression_block_size = kwargs.pop('compression_block_size',1<<22)
blosc_block_size = kwargs.pop('blosc_block_size', 512*1024)
typesize = kwargs.pop('typesize','auto') # dtype size in bytes, e.g. 8 for int64
clevel = kwargs.pop('clevel',1) # compression level, usually only need lowest for zstd
cname = kwargs.pop('cname','zstd') # compressor name, default zstd, good performance/compression tradeoff

shuffle = kwargs.pop('shuffle', 'shuffle')
if shuffle == 'shuffle':
shuffle = blosc.SHUFFLE
Expand All @@ -68,58 +69,58 @@ def compress(self, data, **kwargs):
shuffle = blosc.NOSHUFFLE
else:
raise ValueError(shuffle)

blosc.set_nthreads(nthreads)
blosc.set_blocksize(blosc_block_size)

if typesize == 'auto':
this_typesize = data.itemsize
else:
this_typesize = typesize
#assert this_typesize != 1

nelem = compression_block_size // data.itemsize
for i in range(0,len(data),nelem):
compressed = blosc.compress(data[i:i+nelem], typesize=this_typesize, clevel=clevel, shuffle=shuffle, cname=cname,
**kwargs)
header = struct.pack('!I', len(compressed))
# TODO: this probably triggers a data copy, feels inefficient. Probably have to add output array arg to blosc to fix
yield header + compressed


def decompress(self, blocks, out, **kwargs):
'''Useful decompression kwargs:
nthreads
'''
# TODO: controlled globally for now
#nthreads = kwargs.pop('nthreads',1)
#blosc.set_nthreads(nthreads)

_size = 0
_pos = 0
_buffer = None
_partial_len = b''

decompression_time = 0.
bytesout = 0

# Blosc code probably assumes contiguous buffer
if not out.contiguous:
raise ValueError(out.contiguous)

# get the out address
out = np.frombuffer(out, dtype=np.uint8).ctypes.data

for block in blocks:
block = memoryview(block).cast('c')
try:
block = block.toreadonly() # python>=3.8 only
except AttributeError:
pass

if not block.contiguous:
raise ValueError(block.contiguous)

while len(block):
if not _size:
# Don't know the (compressed) length of this block yet
Expand Down Expand Up @@ -166,14 +167,14 @@ def decompress(self, blocks, out, **kwargs):
_size = 0

return bytesout


class AbacusExtension(Extension):
'''
An ASDF Extension that deals with Abacus types and formats.
Currently only implements Blosc compression.
'''

@property
def extension_uri(self):
"""
Expand All @@ -186,7 +187,7 @@ class itself.
str
"""
return "asdf://abacusnbody.org/extensions/abacus-0.0.1"

@property
def compressors(self):
'''
Expand Down
Loading