Fix/1844 reextract newer archive

[TomCC7]

Contribution path

• Small, safe change that does not need a tracking issue
• Linked issue or discussion: DIM-XXX / #XXX / URL

Problem

Solution

How to Test

AI assistance

Checklist

• I have read and approved the CLA.

[codecov]

❌ 2 Tests Failed:

Tests completed	Failed	Passed	Skipped
3153	2	3151	65

View the top 2 failed test(s) by shortest run time

::dimos.memory2.codecs.test_codecs

Stack Traces | 0s run time

dimos/utils/_git_lfs.py:51: in get_committed_file_sha256
    result = subprocess.run(
        command    = ['git', 'lfs', 'ls-files', '--json', '--include=data/.lfs/go2_short.db.tar.gz', 'HEAD']
        detail     = 'Error: unknown flag: --json\n\n'
        path       = PosixPath('.../dimos/data/.lfs/go2_short.db.tar.gz')
        relative_name = 'data/.lfs/go2_short.db.tar.gz'
        repo_root  = PosixPath('.../_work/dimos/dimos')
../../../../..../uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/subprocess.py:571: in run
    raise CalledProcessError(retcode, process.args,
E   subprocess.CalledProcessError: Command '['git', 'lfs', 'ls-files', '--json', '--include=data/.lfs/go2_short.db.tar.gz', 'HEAD']' returned non-zero exit status 127.
        capture_output = True
        check      = True
        input      = None
        kwargs     = {'cwd': PosixPath('.../_work/dimos/dimos'), 'stderr': -1, 'stdout': -1, 'text': True}
        popenargs  = (['git', 'lfs', 'ls-files', '--json', '--include=data/.lfs/go2_short.db.tar.gz', 'HEAD'],)
        process    = <Popen: returncode: 127 args: ['git', 'lfs', 'ls-files', '--json', '--includ...>
        retcode    = 127
        stderr     = 'Error: unknown flag: --json\n\n'
        stdout     = 'git lfs ls-files [<ref>]\ngit lfs ls-files <ref> <ref>\n\nDisplay paths of Git LFS files that are found in the tree a...atching any of these patterns; see "Fetch settings".\n  \n* -n --name-only:\n  Show only the lfs tracked file names.\n'
        timeout    = None

The above exception was the direct cause of the following exception:
.../memory2/codecs/test_codecs.py:156: in <module>
    for c in [_pickle_case(), _lcm_case(), _lz4_pickle_case(), _lz4_lcm_case(), _jpeg_case()]
        Any        = typing.Any
        Case       = <class 'dimos.memory2.codecs.test_codecs.Case'>
        Codec      = <class 'dimos.memory2.codecs.base.Codec'>
        Image      = <class 'dimos.msgs.sensor_msgs.Image.Image'>
        ImageFormat = <enum 'ImageFormat'>
        JpegCodec  = <class 'dimos.memory2.codecs.jpeg.JpegCodec'>
        LcmCodec   = <class 'dimos.memory2.codecs.lcm.LcmCodec'>
        PickleCodec = <class 'dimos.memory2.codecs.pickle.PickleCodec'>
        PoseStamped = <class 'dimos.msgs.geometry_msgs.PoseStamped.PoseStamped'>
        TYPE_CHECKING = False
        __builtins__ = <builtins>
        __cached__ = '.../_work/dimos/dimos/dimos/memory2/codecs/__pycache__/test_codecs.cpython-312.pyc'
        __doc__    = 'Grid tests for Codec implementations.\n\nRuns roundtrip encode→decode tests across every codec, verifying data preservation.\n'
        __file__   = '.../_work/dimos/dimos/.../memory2/codecs/test_codecs.py'
        __loader__ = <_pytest.assertion.rewrite.AssertionRewritingHook object at 0x72791c95c740>
        __name__   = 'dimos.memory2.codecs.test_codecs'
        __package__ = 'dimos.memory2.codecs'
        __spec__   = ModuleSpec(name='dimos.memory2.codecs.test_codecs', loader=<_pytest.assertion.rewrite.AssertionRewritingHook object at 0x72791c95c740>, origin='.../_work/dimos/dimos/.../memory2/codecs/test_codecs.py')
        _jpeg_case = <function _jpeg_case at 0x72765dc2f380>
        _jpeg_eq   = <function _jpeg_eq at 0x72765dc2f2e0>
        _lcm_case  = <function _lcm_case at 0x72765dc2f100>
        _lcm_values = <function _lcm_values at 0x72765decdf80>
        _lz4_lcm_case = <function _lz4_lcm_case at 0x72765dc2f240>
        _lz4_pickle_case = <function _lz4_pickle_case at 0x72765dc2f1a0>
        _pickle_case = <function _pickle_case at 0x72765dc2e700>
        annotations = _Feature((3, 7, 0, 'beta', 1), None, 16777216)
        codec_for  = <function codec_for at 0x7276f039de40>
        dataclass  = <function dataclass at 0x72791cd7eb60>
        pytest     = <module 'pytest' from '.../_work/dimos/dimos/.venv/lib/python3.12.../site-packages/pytest/__init__.py'>
.../memory2/codecs/test_codecs.py:137: in _jpeg_case
    db_path = get_data("go2_short.db")
        SqliteStore = <class 'dimos.memory2.store.sqlite.SqliteStore'>
        TurboJPEG  = <class 'turbojpeg.TurboJPEG'>
        get_data   = <function get_data at 0x7277c6923a60>
dimos/utils/data.py:388: in get_data
    archive_sha256 = get_committed_file_sha256(archive_file, get_project_root())
        archive_file = PosixPath('.../dimos/data/.lfs/go2_short.db.tar.gz')
        archive_name = 'go2_short.db'
        data_dir   = PosixPath('.../_work/dimos/dimos/data')
        extracted_path = PosixPath('.../_work/dimos/dimos/data/go2_short.db')
        file_path  = PosixPath('.../_work/dimos/dimos/data/go2_short.db')
        name       = 'go2_short.db'
        nested_path = None
        path_parts = ('go2_short.db',)
dimos/utils/_git_lfs.py:60: in get_committed_file_sha256
    raise RuntimeError(
E   RuntimeError: Failed to list committed Git LFS file for .../dimos/data/.lfs/go2_short.db.tar.gz with git lfs ls-files --json --include=data/.lfs/go2_short.db.tar.gz HEAD: Error: unknown flag: --json
        command    = ['git', 'lfs', 'ls-files', '--json', '--include=data/.lfs/go2_short.db.tar.gz', 'HEAD']
        detail     = 'Error: unknown flag: --json\n\n'
        path       = PosixPath('.../dimos/data/.lfs/go2_short.db.tar.gz')
        relative_name = 'data/.lfs/go2_short.db.tar.gz'
        repo_root  = PosixPath('.../_work/dimos/dimos')

dimos.utils.test_data::test_pull_dir

Stack Traces | 0.212s run time

@pytest.mark.self_hosted
    def test_pull_dir() -> None:
        repo_root = data.get_project_root()
        test_dir_name = "ab_lidar_frames"
        test_dir_compressed = data._get_lfs_dir() / (test_dir_name + ".tar.gz")
        test_dir_decompressed = data.get_data_dir() / test_dir_name
    
        # delete decompressed test directory if it exists
        if test_dir_decompressed.exists():
            for item in test_dir_decompressed.iterdir():
                item.unlink()
            test_dir_decompressed.rmdir()
    
        # delete lfs archive file if it exists
        if test_dir_compressed.exists():
            test_dir_compressed.unlink()
    
        # pull the lfs file reference from git
        env = os.environ.copy()
        env["GIT_LFS_SKIP_SMUDGE"] = "1"
        subprocess.run(
            ["git", "checkout", "HEAD", "--", test_dir_compressed],
            cwd=repo_root,
            env=env,
            check=True,
            capture_output=True,
        )
    
        # ensure we have a pointer file from git (small ASCII text file)
        assert test_dir_compressed.exists()
        assert test_dir_compressed.stat().st_size < 200
    
        # trigger a data file pull
        assert data.get_data(test_dir_name) == test_dir_decompressed
        assert test_dir_compressed.stat().st_size > 200
    
        # validate data is received
        assert test_dir_compressed.exists()
        assert test_dir_decompressed.exists()
    
        for [file, expected_hash] in zip(
            sorted(test_dir_decompressed.iterdir()),
            [
                "6c3aaa9a79853ea4a7453c7db22820980ceb55035777f7460d05a0fa77b3b1b3",
                "456cc2c23f4ffa713b4e0c0d97143c27e48bbe6ef44341197b31ce84b3650e74",
            ],
            strict=False,
        ):
            with file.open("rb") as f:
                sha256 = hashlib.sha256(f.read()).hexdigest()
>               assert sha256 == expected_hash
E               AssertionError: assert '3d6398fcc84b...fdb72e6d8649a' == '6c3aaa9a7985...5a0fa77b3b1b3'
E                 
E                 - 6c3aaa9a79853ea4a7453c7db22820980ceb55035777f7460d05a0fa77b3b1b3
E                 + 3d6398fcc84b91b0f325eb41d89b458c96be0601c7c74ccfc85fdb72e6d8649a

env        = {'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjM4ODI2YjE3LTZhMzAtNWY5Yi1iMTY5LThiZWI4MjAyZjcyMyIsIn...2a8233484c.self-hosted-tests.Linux_ghcr_io_dimensional', 'ALIBABA_API_KEY': 'sk-054f2615b3584f81a5959f2e86f96a68', ...}
expected_hash = '6c3aaa9a79853ea4a7453c7db22820980ceb55035777f7460d05a0fa77b3b1b3'
f          = <_io.BufferedReader name='.../data/ab_lidar_frames/.archive_metadata.json'>
file       = PosixPath('.../data/ab_lidar_frames/.archive_metadata.json')
repo_root  = PosixPath('.../__w/dimos/dimos')
sha256     = '3d6398fcc84b91b0f325eb41d89b458c96be0601c7c74ccfc85fdb72e6d8649a'
test_dir_compressed = PosixPath('....../dimos/data/.lfs/ab_lidar_frames.tar.gz')
test_dir_decompressed = PosixPath('....../dimos/data/ab_lidar_frames')
test_dir_name = 'ab_lidar_frames'

dimos/utils/test_data.py:212: AssertionError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}