memos/server/crawler.go

package server

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/labstack/echo/v4"
	"github.com/usememos/memos/plugin/crawler"
	metric "github.com/usememos/memos/plugin/metrics"
)

func (s *Server) registerCrawlerPublicRoutes(g *echo.Group) {
	g.GET("/crawler/website", func(c echo.Context) error {
		ctx := c.Request().Context()
		url := c.QueryParam("url")
		if url == "" {
			return echo.NewHTTPError(http.StatusBadRequest, "Missing website url")
		}

		htmlMeta, err := crawler.GetWebsiteMeta(url)
		if err != nil {
			return echo.NewHTTPError(http.StatusNotAcceptable, fmt.Sprintf("Failed to get website meta with url: %s", url)).SetInternal(err)
		}
		s.Collector.Collect(ctx, &metric.Metric{
			Name: "crawler used",
			Labels: map[string]string{
				"type": "website",
			},
		})

		c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8)
		if err := json.NewEncoder(c.Response().Writer).Encode(composeResponse(htmlMeta)); err != nil {
			return echo.NewHTTPError(http.StatusInternalServerError, "Failed to encode website HTML meta").SetInternal(err)
		}
		return nil
	})
}