Clear-lock quota-counter webid-guard 3 branches merged

config/customise-me.json

@@ -4,6 +4,10 @@
       "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
       "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
     ],
+    "import": [
+      "pivot:config/storage/backend/quota-counter-file.json",
+      "pivot:config/storage/resource-locker/long-expiry.json"
+    ],
     "@graph": [
       {
         "comment": "The settings of your email server.",
@@ -31,18 +35,6 @@
           "templateFolder": "templates/pod"
         }
       },
-      {
-        "comment": "Sets the maximum size of a single pod to 70MB.",
-        "@type": "Override",
-        "overrideInstance": {
-            "@id": "urn:solid-server:default:QuotaStrategy"
-        },
-        "overrideParameters": {            
-            "@type": "PodQuotaStrategy",
-            "limit_amount": 70000000,
-            "limit_unit": "bytes"
-        }
-      },
       {
         "comment": "Serve Databrowser as default representation",
         "@id": "urn:solid-server:default:DefaultUiConverter",

config/dev-http-subdomain.json

@@ -29,14 +29,16 @@
     "css:config/storage/key-value/resource-store.json",
     "css:config/storage/location/pod.json",
     "pivot:config/storage/middleware/default.json",
+    "pivot:config/storage/profile-card-guard.json",
     "css:config/util/auxiliary/acl.json",
     "css:config/util/identifiers/subdomain.json",
 
     "css:config/util/logging/winston.json",
     "css:config/util/representation-conversion/default.json",
     "css:config/util/resource-locker/file.json",
     "css:config/util/variables/default.json",
-    "pivot:config/pivot-overrides.json"
+    "pivot:config/pivot-overrides.json",
+    "pivot:config/pivot-file-locker-overrides.json"
     ],
   "@graph": [
     {

config/dev-http-suffix.json

@@ -29,14 +29,16 @@
     "css:config/storage/key-value/resource-store.json",
     "css:config/storage/location/pod.json",
     "pivot:config/storage/middleware/default.json",
+    "pivot:config/storage/profile-card-guard.json",
     "css:config/util/auxiliary/acl.json",
     "css:config/util/identifiers/suffix.json",
 
     "css:config/util/logging/winston.json",
     "css:config/util/representation-conversion/default.json",
     "css:config/util/resource-locker/file.json",
     "css:config/util/variables/default.json",
-    "pivot:config/pivot-overrides.json"
+    "pivot:config/pivot-overrides.json",
+    "pivot:config/pivot-file-locker-overrides.json"
     ],
   "@graph": [
     {

config/pivot-file-locker-overrides.json

@@ -0,0 +1,20 @@
+{
+  "comment": "Overrides specific to configs that use the file-system resource locker. Imported only by prod/dev configs, NOT by test.json (which uses the memory locker).",
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
+  ],
+  "@graph": [
+    {
+      "comment": "Bound the lock acquisition retries so a contended lock fails fast instead of spinning the event loop and filesystem threadpool forever (~30s max wait).",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:FileSystemResourceLocker" },
+      "overrideParameters": {
+        "@type": "FileSystemResourceLocker",
+        "attemptSettings_retryCount": 600,
+        "attemptSettings_retryDelay": 50,
+        "attemptSettings_retryJitter": 30
+      }
+    }
+  ]
+}

config/pivot-overrides.json

@@ -61,6 +61,82 @@
         "templateEngine": { "@id": "urn:solid-server:default:TemplateEngine" },
         "template": "./templates/main.html.ejs"
       }
+    },
+    {
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:AccountStorage" },
+      "overrideParameters": {
+        "@type": "SafeBaseLoginAccountStorage",
+        "storage": { "@id": "urn:solid-server:default:IndexedStorage" }
+      }
+    },
+    {
+      "comment": "Scope the cookie sweep to its own container so entries()/sweeps do not walk the entire /.internal/ tree (sustained high CPU with many accounts).",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:CookieStorage" },
+      "overrideParameters": {
+        "@type": "WrappedExpiringStorage",
+        "source": {
+          "@type": "MaxKeyLengthStorage",
+          "source": {
+            "@type": "JsonResourceStorage",
+            "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+            "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+            "container": "/.internal/accounts/cookies/"
+          }
+        }
+      }
+    },
+    {
+      "comment": "Scope the forgot-password sweep to its own container.",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:ForgotPasswordStorage" },
+      "overrideParameters": {
+        "@type": "WrappedExpiringStorage",
+        "source": {
+          "@type": "MaxKeyLengthStorage",
+          "source": {
+            "@type": "JsonResourceStorage",
+            "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+            "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+            "container": "/.internal/accounts/forgot-password/"
+          }
+        }
+      }
+    },
+    {
+      "comment": "Scope the token sweep to its own container.",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:ExpiringTokenStorage" },
+      "overrideParameters": {
+        "@type": "WrappedExpiringStorage",
+        "source": {
+          "@type": "MaxKeyLengthStorage",
+          "source": {
+            "@type": "JsonResourceStorage",
+            "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+            "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+            "container": "/.internal/idp/tokens/"
+          }
+        }
+      }
+    },
+    {
+      "comment": "Scope the adapter sweep to its own container. Requires the @id added to the installed CSS adapter-factory.json by the patch-package override.",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:ExpiringAdapterStorage" },
+      "overrideParameters": {
+        "@type": "WrappedExpiringStorage",
+        "source": {
+          "@type": "MaxKeyLengthStorage",
+          "source": {
+            "@type": "JsonResourceStorage",
+            "source": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
+            "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+            "container": "/.internal/idp/adapter/"
+          }
+        }
+      }
     }
   ]
 }

config/prod.json

@@ -29,13 +29,15 @@
     "css:config/storage/key-value/resource-store.json",
     "css:config/storage/location/pod.json",
     "pivot:config/storage/middleware/default.json",
+    "pivot:config/storage/profile-card-guard.json",
     "css:config/util/auxiliary/acl.json",
     "css:config/util/identifiers/subdomain.json",
     "css:config/util/logging/winston.json",
     "css:config/util/representation-conversion/default.json",
     "css:config/util/resource-locker/file.json",
     "css:config/util/variables/default.json",
-    "pivot:config/pivot-overrides.json"
+    "pivot:config/pivot-overrides.json",
+    "pivot:config/pivot-file-locker-overrides.json"
   ],
   "@graph": [
    {
@@ -69,6 +71,11 @@
       "@id": "urn:solid-server:default:ExpiringTokenStorage",
       "@type": "WrappedExpiringStorage",
       "timeout": 1
+    },
+    {
+      "@id": "urn:solid-server:default:ExpiringAdapterStorage",
+      "@type": "WrappedExpiringStorage",
+      "timeout": 1
     }
   ]
 }

config/storage/backend/quota-counter-file.json

@@ -0,0 +1,88 @@
+{
+  "comment": "Design C: incremental per-pod byte counter. QuotaCounter + IncrementalSizeReporter + QuotaDeltaDataAccessor (delta hook) + FastQuotaStrategy. Writes are O(1) after bootstrap; a full du/Node walk only seeds/repairs a pod's counter.",
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
+  ],
+  "@graph": [
+    {
+      "comment": "The shared per-pod counter (in-memory + sidecar + recount engine).",
+      "@id": "urn:solid-server:default:QuotaCounter",
+      "@type": "QuotaCounter",
+      "fileIdentifierMapper": {
+        "@id": "urn:solid-server:default:FileIdentifierMapper"
+      },
+      "rootFilePath": {
+        "@id": "urn:solid-server:default:variable:rootFilePath"
+      },
+      "ignoreFolders": [
+        "(^|/)\\.internal$"
+      ]
+    },
+    {
+      "comment": "SizeReporter backed by the counter: O(1) pod reads, single stat for resources.",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:SizeReporter"
+      },
+      "overrideParameters": {
+        "@type": "IncrementalSizeReporter",
+        "counter": {
+          "@id": "urn:solid-server:default:QuotaCounter"
+        }
+      }
+    },
+    {
+      "comment": "Delta hook: wraps the accessor chain, feeds per-write deltas to the counter. Preserves the content-length filter.",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:FileDataAccessor"
+      },
+      "overrideParameters": {
+        "@type": "QuotaDeltaDataAccessor",
+        "fileIdentifierMapper": {
+          "@id": "urn:solid-server:default:FileIdentifierMapper"
+        },
+        "identifierStrategy": {
+          "@id": "urn:solid-server:default:IdentifierStrategy"
+        },
+        "counter": {
+          "@id": "urn:solid-server:default:QuotaCounter"
+        },
+        "accessor": {
+          "@type": "FilterMetadataDataAccessor",
+          "accessor": {
+            "@id": "urn:solid-server:default:ValidatingFileDataAccessor"
+          },
+          "filters": [
+            {
+              "@type": "FilterPattern",
+              "predicate": "http://www.w3.org/2011/http-headers#content-length"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "comment": "Pod quota strategy that computes the available space once per write (no per-chunk walk).",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:QuotaStrategy"
+      },
+      "overrideParameters": {
+        "@type": "FastQuotaStrategy",
+        "limit_amount": 70000000,
+        "limit_unit": "bytes",
+        "reporter": {
+          "@id": "urn:solid-server:default:SizeReporter"
+        },
+        "identifierStrategy": {
+          "@id": "urn:solid-server:default:IdentifierStrategy"
+        },
+        "accessor": {
+          "@id": "urn:solid-server:default:AtomicFileDataAccessor"
+        }
+      }
+    }
+  ]
+}

config/storage/backend/quota-fast-file.json

@@ -0,0 +1,50 @@
+{
+  "comment": "Fast per-pod quota: DuSizeReporter (du + TTL cache) and FastQuotaStrategy (no per-chunk pod walk).",
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
+  ],
+  "@graph": [
+    {
+      "comment": "SizeReporter backed by du with a per-path TTL cache, measuring apparent bytes (portable, user-manageable). Falls back to the Node walk when du is unavailable.",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:SizeReporter"
+      },
+      "overrideParameters": {
+        "@type": "DuSizeReporter",
+        "fileIdentifierMapper": {
+          "@id": "urn:solid-server:default:FileIdentifierMapper"
+        },
+        "rootFilePath": {
+          "@id": "urn:solid-server:default:variable:rootFilePath"
+        },
+        "ignoreFolders": [
+          "^/\\.internal$"
+        ],
+        "ttl": 5000
+      }
+    },
+    {
+      "comment": "Pod quota strategy that computes the available space once per write instead of per stream chunk.",
+      "@type": "Override",
+      "overrideInstance": {
+        "@id": "urn:solid-server:default:QuotaStrategy"
+      },
+      "overrideParameters": {
+        "@type": "FastQuotaStrategy",
+        "limit_amount": 70000000,
+        "limit_unit": "bytes",
+        "reporter": {
+          "@id": "urn:solid-server:default:SizeReporter"
+        },
+        "identifierStrategy": {
+          "@id": "urn:solid-server:default:IdentifierStrategy"
+        },
+        "accessor": {
+          "@id": "urn:solid-server:default:AtomicFileDataAccessor"
+        }
+      }
+    }
+  ]
+}

config/storage/profile-card-guard.json

@@ -0,0 +1,48 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@solid/pivot/^1.0.0/components/context.jsonld"
+  ],
+  "comment": "Guards the profile card of WebIDs registered on this server, keeping the `solid:oidcIssuer` triple.",
+  "@graph": [
+    {
+      "comment": "Guards registered WebID cards from being deleted or losing their `solid:oidcIssuer` triple.",
+      "@id": "urn:solid-server:default:ResourceStore_CardGuard",
+      "@type": "ProfileCardGuard",
+      "source": { "@id": "urn:solid-server:default:ResourceStore_Converting" },
+      "webIdStore": { "@id": "urn:solid-server:default:WebIdStore" },
+      "baseUrl": { "@id": "urn:solid-server:default:variable:baseUrl" },
+      "converter": { "@id": "urn:solid-server:default:RepresentationConverter" },
+      "relativeWebIdPaths": [
+        "/profile/card#me"
+      ]
+    },
+    {
+      "comment": "WebID store that also exposes whether a WebID is registered (hasWebId), as required by the card guard.",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:WebIdStore" },
+      "overrideParameters": {
+        "@type": "GuardedWebIdStore",
+        "storage": { "@id": "urn:solid-server:default:AccountStorage" }
+      }
+    },
+    {
+      "comment": "Insert the card guard between pivot's RDF patching store and the converting store, so PUT/POST as well as PATCH results are all validated.",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:ResourceStore_RdfPatching" },
+      "overrideParameters": {
+        "@type": "RdfPatchingStore",
+        "source": { "@id": "urn:solid-server:default:ResourceStore_CardGuard" }
+      }
+    },
+    {
+      "comment": "Route the internal key-value storage (accounts, WebID links, ...) through the converting store directly, bypassing the guard.",
+      "@type": "Override",
+      "overrideInstance": { "@id": "urn:solid-server:default:JsonResourceStorage" },
+      "overrideParameters": {
+        "@type": "JsonResourceStorage",
+        "source": { "@id": "urn:solid-server:default:ResourceStore_Converting" }
+      }
+    }
+  ]
+}