From 992d4b14429842cff2aeaaabdc5411fbbfd73094 Mon Sep 17 00:00:00 2001
From: Andreas Kloeckner <inform@tiker.net>
Date: Thu, 17 Sep 2020 21:30:38 -0500
Subject: [PATCH] Avoid directory count limits in persistent_dict
---
pytools/persistent_dict.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/pytools/persistent_dict.py b/pytools/persistent_dict.py
index cad339c..98e944e 100644
--- a/pytools/persistent_dict.py
+++ b/pytools/persistent_dict.py
@@ -166,12 +166,8 @@ class ItemDirManager(CleanupBase):
raise
def mkdir(self):
- from os import mkdir
- try:
- mkdir(self.path)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
+ from os import makedirs
+ makedirs(self.path, exist_ok=True)
def clean_up(self):
pass
@@ -432,7 +428,7 @@ class _PersistentDictBase(object):
import appdirs
container_dir = join(
appdirs.user_cache_dir("pytools", "pytools"),
- "pdict-v2-%s-py%s" % (
+ "pdict-v3-%s-py%s" % (
identifier,
".".join(str(i) for i in sys.version_info),))
@@ -468,7 +464,14 @@ class _PersistentDictBase(object):
def _item_dir(self, hexdigest_key):
from os.path import join
- return join(self.container_dir, hexdigest_key)
+ # Some file systems limit the number of directories in a directory.
+ # For ext4, that limit appears to be 64K for example.
+ # This doesn't solve that problem, but it makes it much less likely
+
+ return join(self.container_dir,
+ hexdigest_key[:3],
+ hexdigest_key[3:6],
+ hexdigest_key[6:])
def _key_file(self, hexdigest_key):
from os.path import join
--
GitLab