fix: Fix non-ASCII character corruption in FileSystemStorageClient on systems without UTF-8 default encoding (#1580)

### Description

- Fix non-ASCII character corruption in `FileSystemStorageClient` on
systems without UTF-8 default encoding

### Issues

- Closes: #1579

### Testing

- Add new tests for storage

Author: Mantisus

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -134,7 +134,7 @@ async def open(
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = DatasetMetadata(**file_content)
@@ -163,7 +163,7 @@ async def open(
 
             # If the dataset directory exists, reconstruct the client from the metadata file.
             if path_to_dataset.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -133,7 +133,7 @@ async def open(
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = KeyValueStoreMetadata(**file_content)
@@ -162,7 +162,7 @@ async def open(
 
             # If the key-value store directory exists, reconstruct the client from the metadata file.
             if path_to_kvs.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:
@@ -239,7 +239,7 @@ async def get_value(self, *, key: str) -> KeyValueStoreRecord | None:
         # Read the metadata file
         async with self._lock:
             try:
-                file = await asyncio.to_thread(open, record_metadata_filepath)
+                file = await asyncio.to_thread(open, record_metadata_filepath, 'r', encoding='utf-8')
             except FileNotFoundError:
                 logger.warning(f'Metadata file disappeared for key "{key}", aborting get_value')
                 return None

src/crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -197,7 +197,7 @@ async def open(
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = RequestQueueMetadata(**file_content)
@@ -232,7 +232,7 @@ async def open(
 
             # If the RQ directory exists, reconstruct the client from the metadata file.
             if path_to_rq.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:
@@ -775,7 +775,7 @@ async def _parse_request_file(cls, file_path: Path) -> Request | None:
         """
         # Open the request file.
         try:
-            file = await asyncio.to_thread(open, file_path)
+            file = await asyncio.to_thread(open, file_path, 'r', encoding='utf-8')
         except FileNotFoundError:
             logger.warning(f'Request file "{file_path}" not found.')
             return None

tests/unit/storages/test_dataset.py

@@ -1082,3 +1082,20 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val
     else:
         with pytest.raises(ValueError, match=rf'Invalid storage name "{name}".*'):
             await Dataset.open(name=name, storage_client=storage_client)
+
+
+async def test_record_with_noascii_chars(dataset: Dataset) -> None:
+    """Test handling record with non-ASCII characters."""
+    init_value = {
+        'record_1': 'Supermaxi El Jardín',
+        'record_2': 'záznam dva',
+        'record_3': '記録三',
+    }
+
+    # Save the record to the dataset
+    await dataset.push_data(init_value)
+
+    # Get the record and verify
+    value = await dataset.get_data()
+    assert value is not None
+    assert value.items[0] == init_value

tests/unit/storages/test_key_value_store.py

@@ -1132,3 +1132,21 @@ async def test_get_auto_saved_value_various_global_clients(
     await kvs.persist_autosaved_values()
 
     assert await kvs.get_value(test_key) == autosaved_value_kvs
+
+
+async def test_record_with_noascii_chars(kvs: KeyValueStore) -> None:
+    """Test storing and retrieving a record with non-ASCII characters."""
+    init_value = {
+        'record_1': 'Supermaxi El Jardín',
+        'record_2': 'záznam dva',
+        'record_3': '記録三',
+    }
+    key = 'non_ascii_key'
+
+    # Save the record in the key-value store
+    await kvs.set_value(key, init_value)
+
+    # Get the record and verify
+    value = await kvs.get_value(key)
+    assert value is not None
+    assert value == init_value

tests/unit/storages/test_request_queue.py

@@ -1348,3 +1348,22 @@ async def test_reclaim_request_with_change_state(rq: RequestQueue) -> None:
     assert reclaimed_request is not None
     assert reclaimed_request.url == 'https://example.com/original'
     assert reclaimed_request.user_data['state'] == 'modified'
+
+
+async def test_request_with_noascii_chars(rq: RequestQueue) -> None:
+    """Test handling requests with non-ASCII characters in user data."""
+    data_with_special_chars = {
+        'record_1': 'Supermaxi El Jardín',
+        'record_2': 'záznam dva',
+        'record_3': '記録三',
+    }
+    init_request = Request.from_url('https://crawlee.dev', user_data=data_with_special_chars)
+
+    # Add a request with special user data
+    await rq.add_request(init_request)
+
+    # Get the request and verify
+    request = await rq.fetch_next_request()
+    assert request is not None
+    assert request.url == 'https://crawlee.dev'
+    assert request.user_data == init_request.user_data