import events work, styling, checkWhitelist refactor

app/src/api/importEvents.go

@@ -40,12 +40,19 @@ func ImportEvents(w http.ResponseWriter, r *http.Request) {
 
 		for _, url := range urls {
 			var events []map[string]interface{}
-			events, err = fetchEventsFromRelay(pubkey, url)
+			var lastEventCreatedAt int64 = 0 // Track the timestamp of the last event fetched
+
+			for {
+				events, err = fetchEventsFromRelay(pubkey, url, lastEventCreatedAt)
 				if err != nil {
 					errorChan <- fmt.Errorf("error fetching events from relay %s: %w", url, err)
 					return
 				}
 
+				if len(events) == 0 {
+					break
+				}
+
 				err = sendEventsToRelay(events)
 				if err != nil {
 					errorChan <- fmt.Errorf("error sending events to relay: %w", err)
@@ -53,6 +60,10 @@ func ImportEvents(w http.ResponseWriter, r *http.Request) {
 				}
 
 				totalEvents += len(events)
+
+				// Update lastEventCreatedAt with the timestamp of the last event fetched
+				lastEventCreatedAt = int64(events[len(events)-1]["created_at"].(float64))
+			}
 		}
 
 		totalEventsChan <- totalEvents
@@ -63,7 +74,7 @@ func ImportEvents(w http.ResponseWriter, r *http.Request) {
 		renderResult(w, true, "Events imported successfully", totalEvents)
 	case err := <-errorChan:
 		renderResult(w, false, err.Error(), 0)
-	case <-time.After(5 * time.Minute):
+	case <-time.After(10 * time.Minute): // Increase timeout for large imports
 		renderResult(w, false, "Timeout importing events", 0)
 	}
 }
@@ -92,7 +103,7 @@ func renderResult(w http.ResponseWriter, success bool, message string, count int
 	}
 }
 
-func fetchEventsFromRelay(pubkey, relayUrl string) ([]map[string]interface{}, error) {
+func fetchEventsFromRelay(pubkey, relayUrl string, lastEventCreatedAt int64) ([]map[string]interface{}, error) {
 	log.Printf("Connecting to relay: %s", relayUrl)
 	conn, err := websocket.Dial(relayUrl, "", "http://localhost/")
 	if err != nil {
@@ -102,7 +113,17 @@ func fetchEventsFromRelay(pubkey, relayUrl string) ([]map[string]interface{}, er
 	defer conn.Close()
 	log.Printf("Connected to relay: %s", relayUrl)
 
-	reqMessage := fmt.Sprintf(`["REQ", "import-sub", {"authors": ["%s"]}]`, pubkey)
+	filters := map[string]interface{}{
+		"authors": []string{pubkey},
+		"limit":   100,
+	}
+
+	if lastEventCreatedAt > 0 {
+		filters["until"] = lastEventCreatedAt - 1
+	}
+
+	filtersJSON, _ := json.Marshal(filters)
+	reqMessage := fmt.Sprintf(`["REQ", "import-sub", %s]`, filtersJSON)
 	log.Printf("Sending request: %s", reqMessage)
 	if _, err := conn.Write([]byte(reqMessage)); err != nil {
 		log.Printf("Error sending request to relay %s: %v", relayUrl, err)
@@ -156,7 +177,7 @@ func sendEventsToRelay(events []map[string]interface{}) error {
 
 	relayUrl := fmt.Sprintf("ws://localhost%s", cfg.Server.Port)
 
-	batchSize := 5 // Reduce the batch size to avoid connection issues
+	batchSize := 20 // Reduce the batch size to avoid connection issues
 	for i := 0; i < len(events); i += batchSize {
 		end := i + batchSize
 		if end > len(events) {
@@ -167,6 +188,9 @@ func sendEventsToRelay(events []map[string]interface{}) error {
 		if err := sendBatchToRelay(batch, relayUrl); err != nil {
 			return err
 		}
+
+		// Wait for a short period to avoid overloading the relay server
+		time.Sleep(1 * time.Second)
 	}
 
 	return nil
@@ -196,9 +220,8 @@ func sendBatchToRelay(events []map[string]interface{}, relayUrl string) error {
 		}
 		log.Printf("Sent event to local relay: %s", event["id"])
 	}
-
 	// Wait for a short period to avoid overloading the relay server
-	time.Sleep(100 * time.Millisecond)
+	time.Sleep(1 * time.Second)
 
 	return nil
 }

app/src/app.go

@@ -23,7 +23,7 @@ func RootHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	data := PageData{
-		Title:  "GRAIN Relay",
+		Title:  "GRAIN Dashboard",
 		Events: events,
 	}
 

app/views/importEvents.html

@@ -1,6 +1,6 @@
 {{define "view"}}
 <main class="flex flex-col items-center justify-center p-8">
-  <div class="mb-4">You are now viewing the {{.Title}}</div>
+  <div class="mb-4">{{.Title}}</div>
   <div
     class="container flex justify-center p-4 border-2 border-gray-600 border-solid rounded-md w-fit"
   >
@@ -10,6 +10,7 @@
       hx-target="#result"
       hx-indicator="#spinner"
     >
+      <div class="content-right">
         <div>
           <label for="pubkey">Pubkey:</label>
           <input
@@ -31,15 +32,25 @@
             required
           />
         </div>
+      </div>
       <button
         class="p-2 m-2 font-bold bg-green-500 rounded-md font-xl"
         type="submit"
       >
         Import Events
       </button>
+      <div class="font-bold text-md">
+        ⚠️ This Feature is Experimental<br />
+        If you are whitelisted, this SHOULD capture all of your events<br />
+        Please Be Patient. Imports can take quite some time due to Rate Limits
+      </div>
     </form>
   </div>
-  <div id="spinner" class="spinner"></div>
+  <div
+    id="spinner"
+    class="m-4 text-lg font-bold text-center spinner"
+    style="display: none"
+  ></div>
   <div id="result" class="p-2 m-2 text-xl font-bold"></div>
   <button
     hx-get="/"
@@ -56,9 +67,11 @@
         document.getElementById("spinner").style.display = "block";
       });
 
-    document
-      .getElementById("result")
-      .addEventListener("htmx:afterRequest", function () {
+    document.addEventListener("htmx:afterRequest", function () {
+      document.getElementById("spinner").style.display = "none";
+    });
+
+    document.addEventListener("htmx:requestError", function () {
       document.getElementById("spinner").style.display = "none";
     });
   </script>

app/views/index.html

@@ -15,7 +15,7 @@
   <button
     hx-get="/import-events"
     hx-swap="outerHTML"
-    class="p-2 mb-2 text-white bg-orange-400 rounded-md"
+    class="p-2 mb-2 text-white bg-purple-500 rounded-md"
     hx-target="body"
   >
     Import Events

app/views/templates/footer.html

@@ -1,5 +1,23 @@
 {{define "footer"}}
 <footer class="text-textMuted">
-  <p>&copy; 2024 GRAIN 🌾, made with 💜 by OceanSlim</p>
+  <p>
+    &copy; 2024
+    <a href="https://github.com/0ceanSlim/grain" class="text-purple-400"
+      >GRAIN 🌾</a
+    >, made with 💜 by
+    <a
+      href="https://njump.me/npub1zmc6qyqdfnllhnzzxr5wpepfpnzcf8q6m3jdveflmgruqvd3qa9sjv7f60"
+      class="text-purple-400"
+      >OceanSlim</a
+    >
+  </p>
+  <p>
+    GRAIN is Proudly Open Source. Don't hesitate to
+    <a
+      href="https://github.com/0ceanSlim/grain?tab=readme-ov-file#development"
+      class="text-purple-400"
+      >contribute</a
+    >!
+  </p>
 </footer>
 {{end}}

app/views/templates/layout.html

@@ -29,10 +29,10 @@
     <style>
       .spinner {
         display: none;
-        width: 50px;
-        height: 50px;
-        border: 5px solid lightgray;
-        border-top: 5px solid blue;
+        width: 32px;
+        height: 32px;
+        border: 5px solid purple;
+        border-top: 5px solid violet;
         border-radius: 50%;
         animation: spin 1s linear infinite;
       }

server/handlers/event.go

@@ -9,7 +9,6 @@ import (
 	"grain/server/handlers/kinds"
 	"grain/server/handlers/response"
 	"grain/server/utils"
-	"strconv"
 
 	relay "grain/server/types"
 
@@ -74,13 +73,13 @@ func HandleKind(ctx context.Context, evt relay.Event, ws *websocket.Conn, eventS
 	}
 
 	// Check if the kind is whitelisted
-	if config.GetConfig().KindWhitelist.Enabled && !isKindWhitelisted(evt.Kind) {
+	if config.GetConfig().KindWhitelist.Enabled && !utils.IsKindWhitelisted(evt.Kind) {
 		response.SendOK(ws, evt.ID, false, "not allowed: event kind is not whitelisted")
 		return
 	}
 
 	// Check pubkey/npub whitelist only if the kind is not whitelisted
-	if config.GetConfig().PubkeyWhitelist.Enabled && !isPubKeyWhitelisted(evt.PubKey) {
+	if config.GetConfig().PubkeyWhitelist.Enabled && !utils.IsPubKeyWhitelisted(evt.PubKey) {
 		response.SendOK(ws, evt.ID, false, "not allowed: pubkey or npub is not whitelisted")
 		return
 	}
@@ -148,52 +147,3 @@ func determineCategory(kind int) string {
 	}
 }
 
-// Helper function to check if a pubkey or npub is whitelisted
-func isPubKeyWhitelisted(pubKey string) bool {
-	cfg := config.GetConfig()
-	if !cfg.PubkeyWhitelist.Enabled {
-		return true
-	}
-
-	// Check pubkeys
-	for _, whitelistedKey := range cfg.PubkeyWhitelist.Pubkeys {
-		if pubKey == whitelistedKey {
-			return true
-		}
-	}
-
-	// Check npubs
-	for _, npub := range cfg.PubkeyWhitelist.Npubs {
-		decodedPubKey, err := utils.DecodeNpub(npub)
-		if err != nil {
-			fmt.Println("Error decoding npub:", err)
-			continue
-		}
-		if pubKey == decodedPubKey {
-			return true
-		}
-	}
-
-	return false
-}
-
-func isKindWhitelisted(kind int) bool {
-	cfg := config.GetConfig()
-	if !cfg.KindWhitelist.Enabled {
-		return true
-	}
-
-	// Check event kinds
-	for _, whitelistedKindStr := range cfg.KindWhitelist.Kinds {
-		whitelistedKind, err := strconv.Atoi(whitelistedKindStr)
-		if err != nil {
-			fmt.Println("Error converting whitelisted kind to int:", err)
-			continue
-		}
-		if kind == whitelistedKind {
-			return true
-		}
-	}
-
-	return false
-}

server/utils/checkWhitelist.go

@@ -0,0 +1,57 @@
+package utils
+
+import (
+	"fmt"
+	"grain/config"
+	"strconv"
+)
+
+// Helper function to check if a pubkey or npub is whitelisted
+func IsPubKeyWhitelisted(pubKey string) bool {
+	cfg := config.GetConfig()
+	if !cfg.PubkeyWhitelist.Enabled {
+		return true
+	}
+
+	// Check pubkeys
+	for _, whitelistedKey := range cfg.PubkeyWhitelist.Pubkeys {
+		if pubKey == whitelistedKey {
+			return true
+		}
+	}
+
+	// Check npubs
+	for _, npub := range cfg.PubkeyWhitelist.Npubs {
+		decodedPubKey, err := DecodeNpub(npub)
+		if err != nil {
+			fmt.Println("Error decoding npub:", err)
+			continue
+		}
+		if pubKey == decodedPubKey {
+			return true
+		}
+	}
+
+	return false
+}
+
+func IsKindWhitelisted(kind int) bool {
+	cfg := config.GetConfig()
+	if !cfg.KindWhitelist.Enabled {
+		return true
+	}
+
+	// Check event kinds
+	for _, whitelistedKindStr := range cfg.KindWhitelist.Kinds {
+		whitelistedKind, err := strconv.Atoi(whitelistedKindStr)
+		if err != nil {
+			fmt.Println("Error converting whitelisted kind to int:", err)
+			continue
+		}
+		if kind == whitelistedKind {
+			return true
+		}
+	}
+
+	return false
+}