From e4f44a66cdffd15dc207241985e13019b6df760e Mon Sep 17 00:00:00 2001 From: Chris kerr Date: Sat, 16 Nov 2024 10:42:35 -0500 Subject: [PATCH] fixed event purging --- config/Whitelist.go | 138 +++++++++++++++++---------------- server/db/mongo/purgeEvents.go | 51 ++++++++---- 2 files changed, 109 insertions(+), 80 deletions(-) diff --git a/config/Whitelist.go b/config/Whitelist.go index 83a12cc..92a7eef 100644 --- a/config/Whitelist.go +++ b/config/Whitelist.go @@ -10,86 +10,92 @@ import ( // CheckWhitelist checks if an event meets the whitelist criteria. func CheckWhitelist(evt nostr.Event) (bool, string) { - whitelistCfg := GetWhitelistConfig() - if whitelistCfg == nil { - return false, "Internal server error: whitelist configuration is missing" - } + whitelistCfg := GetWhitelistConfig() + if whitelistCfg == nil { + return false, "Internal server error: whitelist configuration is missing" + } - // Check if the event's kind is whitelisted - if whitelistCfg.KindWhitelist.Enabled && !IsKindWhitelisted(evt.Kind) { - return false, "not allowed: event kind is not whitelisted" - } + // Check if the event's kind is whitelisted + if whitelistCfg.KindWhitelist.Enabled && !IsKindWhitelisted(evt.Kind) { + return false, "not allowed: event kind is not whitelisted" + } - // Check if the event's pubkey is whitelisted - if whitelistCfg.PubkeyWhitelist.Enabled && !IsPubKeyWhitelisted(evt.PubKey) { - return false, "not allowed: pubkey or npub is not whitelisted" - } + // Check if the event's pubkey is whitelisted + if whitelistCfg.PubkeyWhitelist.Enabled && !IsPubKeyWhitelisted(evt.PubKey, false) { + return false, "not allowed: pubkey or npub is not whitelisted" + } - return true, "" + return true, "" } // IsPubKeyWhitelisted checks if a pubkey or npub is whitelisted, considering pubkeys from domains. -func IsPubKeyWhitelisted(pubKey string) bool { - cfg := GetWhitelistConfig() - if cfg == nil || !cfg.PubkeyWhitelist.Enabled { - return true // Whitelisting is not enforced if the configuration is missing or disabled - } +// The `forPurge` flag indicates if the check is for purging purposes. +func IsPubKeyWhitelisted(pubKey string, forPurge bool) bool { + cfg := GetWhitelistConfig() + if cfg == nil { + return false // No configuration means no whitelisting. + } - // Check statically defined pubkeys - for _, whitelistedKey := range cfg.PubkeyWhitelist.Pubkeys { - if pubKey == whitelistedKey { - return true - } - } + // If the whitelist is disabled but this check is for purging, we still evaluate it. + if !cfg.PubkeyWhitelist.Enabled && !forPurge { + return true // Whitelisting is not enforced for posting if disabled. + } - // Check statically defined npubs after decoding them to pubkeys - for _, npub := range cfg.PubkeyWhitelist.Npubs { - decodedPubKey, err := utils.DecodeNpub(npub) - if err != nil { - log.Printf("Error decoding npub: %v", err) - continue - } - if pubKey == decodedPubKey { - return true - } - } + // Check statically defined pubkeys + for _, whitelistedKey := range cfg.PubkeyWhitelist.Pubkeys { + if pubKey == whitelistedKey { + return true + } + } - // Fetch and check pubkeys from domains if domain whitelisting is enabled - if cfg.DomainWhitelist.Enabled { - domains := cfg.DomainWhitelist.Domains - pubkeys, err := utils.FetchPubkeysFromDomains(domains) - if err != nil { - log.Printf("Error fetching pubkeys from domains: %v", err) - return false // Consider returning true or handling based on your application's needs - } + // Check statically defined npubs after decoding them to pubkeys + for _, npub := range cfg.PubkeyWhitelist.Npubs { + decodedPubKey, err := utils.DecodeNpub(npub) + if err != nil { + log.Printf("Error decoding npub: %v", err) + continue + } + if pubKey == decodedPubKey { + return true + } + } - for _, domainPubKey := range pubkeys { - if pubKey == domainPubKey { - return true - } - } - } + // Fetch and check pubkeys from domains if domain whitelisting is enabled + if cfg.DomainWhitelist.Enabled { + domains := cfg.DomainWhitelist.Domains + pubkeys, err := utils.FetchPubkeysFromDomains(domains) + if err != nil { + log.Printf("Error fetching pubkeys from domains: %v", err) + return false // Consider errors as non-whitelisted for purging + } - return false + for _, domainPubKey := range pubkeys { + if pubKey == domainPubKey { + return true + } + } + } + + return false // Not whitelisted } // Check if a kind is whitelisted func IsKindWhitelisted(kind int) bool { - cfg := GetWhitelistConfig() - if !cfg.KindWhitelist.Enabled { - return true - } + cfg := GetWhitelistConfig() + if !cfg.KindWhitelist.Enabled { + return true + } - for _, whitelistedKindStr := range cfg.KindWhitelist.Kinds { - whitelistedKind, err := strconv.Atoi(whitelistedKindStr) - if err != nil { - fmt.Println("Error converting whitelisted kind to int:", err) - continue - } - if kind == whitelistedKind { - return true - } - } + for _, whitelistedKindStr := range cfg.KindWhitelist.Kinds { + whitelistedKind, err := strconv.Atoi(whitelistedKindStr) + if err != nil { + fmt.Println("Error converting whitelisted kind to int:", err) + continue + } + if kind == whitelistedKind { + return true + } + } - return false -} \ No newline at end of file + return false +} diff --git a/server/db/mongo/purgeEvents.go b/server/db/mongo/purgeEvents.go index 9a3c305..8e77515 100644 --- a/server/db/mongo/purgeEvents.go +++ b/server/db/mongo/purgeEvents.go @@ -21,8 +21,14 @@ func PurgeOldEvents(cfg *types.EventPurgeConfig) { } client := GetClient() - cutoff := time.Now().Add(-time.Duration(cfg.KeepIntervalHours) * time.Hour).Unix() + + // Calculate the cutoff time + currentTime := time.Now().Unix() + cutoff := currentTime - int64(cfg.KeepIntervalHours*3600) // Convert hours to seconds + var collectionsToPurge []string + totalPurged := 0 + totalKept := 0 // Determine collections to purge if cfg.PurgeByKindEnabled { @@ -30,7 +36,6 @@ func PurgeOldEvents(cfg *types.EventPurgeConfig) { collectionsToPurge = append(collectionsToPurge, "event-kind"+strconv.Itoa(kind)) } } else { - // If `purge_by_kind_enabled` is false, add all potential event kinds or find dynamically collectionsToPurge = getAllEventCollections(client) } @@ -49,27 +54,45 @@ func PurgeOldEvents(cfg *types.EventPurgeConfig) { var evt nostr.Event if err := cursor.Decode(&evt); err != nil { log.Printf("Error decoding event from %s: %v", collectionName, err) + totalKept++ continue } - // Skip if the pubkey is whitelisted - if cfg.ExcludeWhitelisted && config.IsPubKeyWhitelisted(evt.PubKey) { - log.Printf("Skipping purging for whitelisted event ID: %s, pubkey: %s", evt.ID, evt.PubKey) + // Debug log to check created_at and cutoff + //log.Printf("Processing event ID: %s, pubkey: %s, created_at: %d, cutoff: %d", evt.ID, evt.PubKey, evt.CreatedAt, cutoff) + + // If the event is not older than the cutoff, mark it as kept + if evt.CreatedAt >= cutoff { + totalKept++ continue } - // Check if purging by category is enabled and if the event matches the allowed category + // Skip purging if the pubkey is whitelisted + if cfg.ExcludeWhitelisted && config.IsPubKeyWhitelisted(evt.PubKey, true) { + //log.Printf("Event ID: %s is kept because the pubkey is whitelisted.", evt.ID) + totalKept++ + continue + } + + // Check if purging by category is enabled and matches the event's category category := utils.DetermineEventCategory(evt.Kind) - if purge, exists := cfg.PurgeByCategory[category]; exists && purge { - _, err := collection.DeleteOne(context.TODO(), bson.M{"id": evt.ID}) - if err != nil { - log.Printf("Error purging event ID %s from %s: %v", evt.ID, collectionName, err) - } else { - log.Printf("Purged event ID: %s from %s", evt.ID, collectionName) - } + if purge, exists := cfg.PurgeByCategory[category]; !exists || !purge { + totalKept++ + continue + } + + // Proceed to delete the event + _, err = collection.DeleteOne(context.TODO(), bson.M{"id": evt.ID}) + if err != nil { + log.Printf("Error purging event ID %s from %s: %v", evt.ID, collectionName, err) + totalKept++ + } else { + totalPurged++ } } } + + log.Printf("Purging completed: Total events purged = %d, Total events kept = %d", totalPurged, totalKept) } // getAllEventCollections returns a list of all event collections if purging all kinds. @@ -97,6 +120,6 @@ func ScheduleEventPurging(cfg *types.ServerConfig) { for range ticker.C { PurgeOldEvents(&cfg.EventPurge) - log.Println("Scheduled purging completed.") + //log.Println("Scheduled purging completed.") } }