backup relay configuration added

app/src/api/importEvents.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"grain/config"
+	nostr "grain/server/types"
 
 	"golang.org/x/net/websocket"
 )
@@ -53,13 +54,20 @@ func ImportEvents(w http.ResponseWriter, r *http.Request) {
 					break
 				}
 
-				err = sendEventsToRelay(events)
+				// Filter events based on the whitelist
+				whitelistedEvents := filterWhitelistedEvents(events)
+				if len(whitelistedEvents) == 0 {
+					log.Printf("No whitelisted events to import from relay %s", url)
+					break
+				}
+
+				err = sendEventsToRelay(whitelistedEvents)
 				if err != nil {
 					errorChan <- fmt.Errorf("error sending events to relay: %w", err)
 					return
 				}
 
-				totalEvents += len(events)
+				totalEvents += len(whitelistedEvents)
 
 				// Update lastEventCreatedAt with the timestamp of the last event fetched
 				lastEventCreatedAt = int64(events[len(events)-1]["created_at"].(float64))
@@ -79,6 +87,40 @@ func ImportEvents(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// filterWhitelistedEvents filters events based on the whitelist configuration.
+func filterWhitelistedEvents(events []map[string]interface{}) []map[string]interface{} {
+	var whitelistedEvents []map[string]interface{}
+
+	// Load the whitelist configuration
+	whitelistCfg := config.GetWhitelistConfig()
+	if whitelistCfg == nil {
+		log.Println("Whitelist configuration is not loaded. Allowing all events.")
+		return events
+	}
+
+	for _, event := range events {
+		evt := nostr.Event{
+			ID:        event["id"].(string),
+			PubKey:    event["pubkey"].(string),
+			CreatedAt: int64(event["created_at"].(float64)),
+			Kind:      int(event["kind"].(float64)),
+			Content:   event["content"].(string),
+			Tags:      event["tags"].([][]string),
+			Sig:       event["sig"].(string),
+		}
+
+		// Check the whitelist criteria
+		isWhitelisted, _ := config.CheckWhitelist(evt)
+		if isWhitelisted {
+			whitelistedEvents = append(whitelistedEvents, event)
+		} else {
+			log.Printf("Event ID %s blocked due to whitelist rules", evt.ID)
+		}
+	}
+
+	return whitelistedEvents
+}
+
 func renderResult(w http.ResponseWriter, success bool, message string, count int) {
 	tmpl, err := template.New("result").Parse(`
 		{{ if .Success }}

app/static/examples/config.example.yml

@@ -14,6 +14,10 @@ server:
   max_connections: 100
   max_subscriptions_per_client: 10
 
+backup_relay:
+  enabled: false # Set to true to enable sending events to the backup relay
+  url: "wss://some-relay.com" # URL of the backup relay
+
 event_time_constraints:
   min_created_at: 1577836800 # January 1, 2020, as Unix timestamp
   # min_created_at_string: now-5m  # Custom value to indicate 5 minutes in the past

config/types/serverConfig.go

@@ -26,4 +26,8 @@ type ServerConfig struct {
 	Auth                 AuthConfig           `yaml:"auth"`
 	EventPurge           EventPurgeConfig     `yaml:"event_purge"`
 	EventTimeConstraints EventTimeConstraints `yaml:"event_time_constraints"` // Added this field
+	BackupRelay          struct {
+		Enabled bool   `yaml:"enabled"`
+		URL     string `yaml:"url"`
+	} `yaml:"backup_relay"`
 }

server/db/mongo/checkDuplicate.go

@@ -0,0 +1,25 @@
+package mongo
+
+import (
+	"context"
+	"fmt"
+	nostr "grain/server/types" // Adjust import path as needed
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// CheckDuplicateEvent checks if an event with the same ID already exists in the collection.
+func CheckDuplicateEvent(ctx context.Context, evt nostr.Event) (bool, error) {
+	collection := GetCollection(evt.Kind)
+	filter := bson.M{"id": evt.ID}
+
+	var existingEvent nostr.Event
+	err := collection.FindOne(ctx, filter).Decode(&existingEvent)
+	if err != nil {
+		if err.Error() == "mongo: no documents in result" {
+			return false, nil // No duplicate found
+		}
+		return false, fmt.Errorf("error checking for duplicate event: %v", err)
+	}
+	return true, nil // Duplicate found
+}

server/db/mongo/purgeEvents.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"grain/config"
 	types "grain/config/types"
-	"grain/server/utils"
+	nostr "grain/server/types"
 	"log"
 	"time"
 
@@ -12,7 +12,7 @@ import (
 )
 
 // PurgeOldEvents removes old events based on the configuration and a list of whitelisted pubkeys.
-func PurgeOldEvents(cfg *types.EventPurgeConfig, whitelist []string) {
+func PurgeOldEvents(cfg *types.EventPurgeConfig) {
 	if !cfg.Enabled {
 		return
 	}
@@ -23,78 +23,49 @@ func PurgeOldEvents(cfg *types.EventPurgeConfig, whitelist []string) {
 	// Calculate the cutoff time
 	cutoff := time.Now().AddDate(0, 0, -cfg.KeepDurationDays).Unix()
 
-	// Create the filter for purging old events
+	// Create the base filter for fetching old events
-	filter := bson.M{
+	baseFilter := bson.M{
-		"created_at": bson.M{"$lt": cutoff}, // Filter older events
+		"created_at": bson.M{"$lt": cutoff}, // Filter for events older than the cutoff
 	}
 
-	// Exclude whitelisted pubkeys if specified in the config
+	cursor, err := collection.Find(context.TODO(), baseFilter)
-	if cfg.ExcludeWhitelisted && len(whitelist) > 0 {
-		filter["pubkey"] = bson.M{"$nin": whitelist} // Exclude whitelisted pubkeys
-	}
-
-	// Handle purging by category
-	for category, purge := range cfg.PurgeByCategory {
-		if purge {
-			filter["category"] = category
-			_, err := collection.DeleteMany(context.TODO(), filter)
 	if err != nil {
-				log.Printf("Error purging events by category %s: %v", category, err)
+		log.Printf("Error fetching old events for purging: %v", err)
-			}
+		return
 	}
+	defer cursor.Close(context.TODO())
+
+	for cursor.Next(context.TODO()) {
+		var evt nostr.Event
+		if err := cursor.Decode(&evt); err != nil {
+			log.Printf("Error decoding event: %v", err)
+			continue
 		}
 
-	// Handle purging by kind
+		// Check if the event's pubkey is whitelisted and skip purging if configured to do so
-	for _, kindRule := range cfg.PurgeByKind {
+		if cfg.ExcludeWhitelisted && config.IsPubKeyWhitelisted(evt.PubKey) {
-		if kindRule.Enabled {
+			log.Printf("Skipping purging for whitelisted event ID: %s, pubkey: %s", evt.ID, evt.PubKey)
-			filter["kind"] = kindRule.Kind
+			continue
-			_, err := collection.DeleteMany(context.TODO(), filter)
-			if err != nil {
-				log.Printf("Error purging events by kind %d: %v", kindRule.Kind, err)
 		}
+
+		// Proceed with deleting the event if it is not whitelisted
+		_, err := collection.DeleteOne(context.TODO(), bson.M{"id": evt.ID})
+		if err != nil {
+			log.Printf("Error purging event ID %s: %v", evt.ID, err)
+		} else {
+			log.Printf("Purged event ID: %s", evt.ID)
 		}
 	}
 }
 
 // ScheduleEventPurging runs the event purging at a configurable interval.
 func ScheduleEventPurging(cfg *types.ServerConfig) {
-	// Use the purge interval from the configuration
 	purgeInterval := time.Duration(cfg.EventPurge.PurgeIntervalHours) * time.Hour
 	ticker := time.NewTicker(purgeInterval)
 	defer ticker.Stop()
 
 	for range ticker.C {
-		// Fetch the whitelisted pubkeys without passing cfg directly
+		PurgeOldEvents(&cfg.EventPurge)
-		whitelist := getWhitelistedPubKeys()
+		log.Println("Scheduled purging completed.")
-		PurgeOldEvents(&cfg.EventPurge, whitelist)
-		log.Printf("Purged old events, keeping whitelisted pubkeys: %v", whitelist)
 	}
 }
-
-// Fetch whitelisted pubkeys from both the whitelist config and any additional domains.
-func getWhitelistedPubKeys() []string {
-	// Get the whitelist configuration
-	whitelistCfg := config.GetWhitelistConfig()
-	if whitelistCfg == nil {
-		log.Println("whitelistCfg is nil, returning an empty list of whitelisted pubkeys.")
-		return []string{}
-	}
-
-	// Start with the statically defined pubkeys
-	whitelistedPubkeys := whitelistCfg.PubkeyWhitelist.Pubkeys
-
-	// Fetch pubkeys from domains if domain whitelist is enabled
-	if whitelistCfg.DomainWhitelist.Enabled {
-		domains := whitelistCfg.DomainWhitelist.Domains
-		pubkeys, err := utils.FetchPubkeysFromDomains(domains)
-		if err != nil {
-			log.Printf("Error fetching pubkeys from domains: %v", err)
-			// Return the existing statically whitelisted pubkeys in case of an error
-			return whitelistedPubkeys
-		}
-		// Append fetched pubkeys from domains to the whitelisted pubkeys
-		whitelistedPubkeys = append(whitelistedPubkeys, pubkeys...)
-	}
-
-	return whitelistedPubkeys
-}

server/handlers/event.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"grain/config"
 	"grain/server/db/mongo"
+	"log"
 	"time"
 
 	"grain/server/handlers/response"
@@ -66,9 +67,66 @@ func HandleEvent(ws *websocket.Conn, message []interface{}) {
 		return
 	}
 
+	// Check for duplicate event
+	isDuplicate, err := mongo.CheckDuplicateEvent(context.TODO(), evt)
+	if err != nil {
+		fmt.Printf("Error checking for duplicate event: %v\n", err)
+		response.SendOK(ws, evt.ID, false, "error: internal server error during duplicate check")
+		return
+	}
+
+	if isDuplicate {
+		response.SendOK(ws, evt.ID, false, "blocked: the database already contains this event")
+		return
+	}
+
 	// Store the event in MongoDB or other storage
 	mongo.StoreMongoEvent(context.TODO(), evt, ws)
 	fmt.Println("Event processed:", evt.ID)
+
+	// Load the config and check for errors
+	cfg, err := config.LoadConfig("config.yml")
+	if err != nil {
+		log.Printf("Error loading configuration: %v", err)
+		return
+	}
+
+	// Send the event to the backup relay if configured
+	if cfg.BackupRelay.Enabled {
+		go func() {
+			err := sendToBackupRelay(cfg.BackupRelay.URL, evt)
+			if err != nil {
+				log.Printf("Failed to send event %s to backup relay: %v", evt.ID, err)
+			} else {
+				log.Printf("Event %s successfully sent to backup relay", evt.ID)
+			}
+		}()
+	}
+}
+
+func sendToBackupRelay(backupURL string, evt nostr.Event) error {
+	conn, err := websocket.Dial(backupURL, "", "http://localhost/")
+	if err != nil {
+		return fmt.Errorf("error connecting to backup relay %s: %w", backupURL, err)
+	}
+	defer conn.Close()
+
+	// Create the message to send
+	eventMessage := []interface{}{"EVENT", evt}
+	eventMessageBytes, err := json.Marshal(eventMessage)
+	if err != nil {
+		return fmt.Errorf("error marshaling event message: %w", err)
+	}
+
+	if _, err := conn.Write(eventMessageBytes); err != nil {
+		return fmt.Errorf("error sending event message to backup relay: %w", err)
+	}
+
+	// Log and return
+	log.Printf("Event %s sent to backup relay %s", evt.ID, backupURL)
+	time.Sleep(500 * time.Millisecond) // Optional: small delay to avoid rapid successive sends
+
+	return nil
 }
 
 // Validate event timestamps against the configured min and max values