event purging configurations

app/static/examples/config.example.yml

@@ -89,3 +89,20 @@ blacklist: #Removing a pubkey from the Blacklist requires a hard restart; Blackl
     - db0c9b8acd6101adb9b281c5321f98f6eebb33c5719d230ed1870997538a9765
   permanent_blacklist_npubs: # List of permanently banned npubs
     - npub1x0r5gflnk2mn6h3c70nvnywpy2j46gzqwg6k7uw6fxswyz0md9qqnhshtn
+
+event_purge:
+  enabled: true # Toggle to enable/disable event purging
+  keep_duration_days: 2 # Number of days to keep events
+  purge_interval_hours: 24 # Runs every 24 hours
+  purge_by_category: # Configure purging based on categories
+    parameterized_replaceable: false
+    regular: true
+    replaceable: false
+  purge_by_kind: # Configure purging based on event kind
+    - kind: 0
+      enabled: false
+    - kind: 1
+      enabled: true
+    - kind: 3
+      enabled: false
+  exclude_whitelisted: true # Exclude events from whitelisted pubkeys during purging

config/types/eventPurging.go

@@ -0,0 +1,15 @@
+package config
+
+type EventPurgeConfig struct {
+	Enabled            bool            `yaml:"enabled"`
+	KeepDurationDays   int             `yaml:"keep_duration_days"`
+	PurgeIntervalHours int             `yaml:"purge_interval_hours"`
+	PurgeByCategory    map[string]bool `yaml:"purge_by_category"`
+	PurgeByKind        []KindPurgeRule `yaml:"purge_by_kind"`
+	ExcludeWhitelisted bool            `yaml:"exclude_whitelisted"`
+}
+
+type KindPurgeRule struct {
+	Kind    int  `yaml:"kind"`
+	Enabled bool `yaml:"enabled"`
+}

config/types/serverConfig.go

@@ -20,4 +20,5 @@ type ServerConfig struct {
 	Blacklist       BlacklistConfig       `yaml:"blacklist"`
 	ResourceLimits  ResourceLimits        `yaml:"resource_limits"`
 	Auth            AuthConfig            `yaml:"auth"`
+	EventPurge      EventPurgeConfig      `yaml:"event_purge"`
 }

main.go

@@ -39,6 +39,9 @@ func main() {
 			log.Fatal("Error loading config: ", err)
 		}
 
+		// Start event purging in the background
+		go mongo.ScheduleEventPurging(cfg)
+
 		config.SetResourceLimit(&cfg.ResourceLimits) // Apply limits once before starting the server
 
 		client, err := mongo.InitDB(cfg)
@@ -70,9 +73,9 @@ func main() {
 
 		case <-signalChan:
 			log.Println("Shutting down server...")
-			server.Close()          // Stop the server
+			server.Close()             // Stop the server
 			mongo.DisconnectDB(client) // Disconnect from MongoDB
-			wg.Wait()               // Wait for all goroutines to finish
+			wg.Wait()                  // Wait for all goroutines to finish
 			return
 		}
 	}

server/db/mongo/purgeEvents.go

@@ -0,0 +1,86 @@
+package mongo
+
+import (
+	"context"
+	types "grain/config/types"
+	"grain/server/utils"
+	"log"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func PurgeOldEvents(cfg *types.EventPurgeConfig, whitelist []string) {
+	if !cfg.Enabled {
+		return
+	}
+
+	client := GetClient()
+	collection := client.Database("grain").Collection("events")
+
+	// Calculate the cutoff time
+	cutoff := time.Now().AddDate(0, 0, -cfg.KeepDurationDays).Unix()
+
+	filter := bson.M{
+		"created_at": bson.M{"$lt": cutoff}, // Filter older events
+	}
+
+	if cfg.ExcludeWhitelisted && len(whitelist) > 0 {
+		filter["pubkey"] = bson.M{"$nin": whitelist} // Exclude whitelisted pubkeys
+	}
+
+	// Handle purging by category
+	for category, purge := range cfg.PurgeByCategory {
+		if purge {
+			filter["category"] = category
+			_, err := collection.DeleteMany(context.TODO(), filter)
+			if err != nil {
+				log.Printf("Error purging events by category %s: %v", category, err)
+			}
+		}
+	}
+
+	// Handle purging by kind
+	for _, kindRule := range cfg.PurgeByKind {
+		if kindRule.Enabled {
+			filter["kind"] = kindRule.Kind
+			_, err := collection.DeleteMany(context.TODO(), filter)
+			if err != nil {
+				log.Printf("Error purging events by kind %d: %v", kindRule.Kind, err)
+			}
+		}
+	}
+}
+
+// Example of a periodic purging task
+// ScheduleEventPurging runs the event purging at a configurable interval.
+func ScheduleEventPurging(cfg *types.ServerConfig) {
+	// Use the purge interval from the configuration
+	purgeInterval := time.Duration(cfg.EventPurge.PurgeIntervalHours) * time.Hour
+	ticker := time.NewTicker(purgeInterval)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		whitelist := getWhitelistedPubKeys(cfg)
+		PurgeOldEvents(&cfg.EventPurge, whitelist)
+	}
+}
+
+// Fetch whitelisted pubkeys from both the config and any additional domains.
+func getWhitelistedPubKeys(cfg *types.ServerConfig) []string {
+	whitelistedPubkeys := cfg.PubkeyWhitelist.Pubkeys
+
+	// Fetch pubkeys from domains if domain whitelist is enabled
+	if cfg.DomainWhitelist.Enabled {
+		domains := cfg.DomainWhitelist.Domains
+		pubkeys, err := utils.FetchPubkeysFromDomains(domains)
+		if err != nil {
+			log.Printf("Error fetching pubkeys from domains: %v", err)
+			return whitelistedPubkeys // Return existing whitelisted pubkeys in case of error
+		}
+		// Append fetched pubkeys from domains to the whitelisted pubkeys
+		whitelistedPubkeys = append(whitelistedPubkeys, pubkeys...)
+	}
+
+	return whitelistedPubkeys
+}