From dd1846589b7a727164c52538a10b89d8689b8a81 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Sat, 25 Sep 2021 00:58:57 +0800 Subject: [PATCH 1/6] refactor(rss): basic concept & cache re-org --- config/config.exs | 27 ++++- lib/groupher_server/application.ex | 48 +++++--- lib/groupher_server/cms/cms.ex | 4 + .../cms/delegates/blog_curd.ex | 77 ++++++++++++ lib/groupher_server/cms/models/blog_rss.ex | 60 ++++++++++ .../cms/models/embeds/blog_author.ex | 26 ++++ .../cms/models/embeds/blog_history_feed.ex | 25 ++++ lib/helper/RSS.ex | 112 ++++++++++++++++++ lib/helper/cache.ex | 47 +++++--- mix.exs | 5 +- mix.lock | 4 +- .../20210924023609_create_blog_rss.exs | 17 +++ .../query/cms/comments/guide_comment_test.exs | 1 - .../cms/comments/meetup_comment_test.exs | 1 - test/helper/rss_test.exs | 41 +++++++ test/support/factory.ex | 9 ++ 16 files changed, 466 insertions(+), 38 deletions(-) create mode 100644 lib/groupher_server/cms/delegates/blog_curd.ex create mode 100644 lib/groupher_server/cms/models/blog_rss.ex create mode 100644 lib/groupher_server/cms/models/embeds/blog_author.ex create mode 100644 lib/groupher_server/cms/models/embeds/blog_history_feed.ex create mode 100644 lib/helper/RSS.ex create mode 100644 priv/repo/migrations/20210924023609_create_blog_rss.exs create mode 100644 test/helper/rss_test.exs diff --git a/config/config.exs b/config/config.exs index 10b2bfb75..266ff0f54 100644 --- a/config/config.exs +++ b/config/config.exs @@ -134,10 +134,24 @@ config :groupher_server, GroupherServer.Mailer, adapter: Bamboo.MailgunAdapter, domain: "mailer.coderplanets.com" -# handle background jobs -config :rihanna, - jobs_table_name: "background_jobs", - producer_postgres_connection: {Ecto, GroupherServer.Repo} +config :groupher_server, :cache, + pool: %{ + common: %{ + name: :common, + size: 5000, + minutes: 10 + }, + user_login: %{ + name: :user_login, + size: 10_000, + minutes: 10_080 + }, + blog_rss: %{ + name: :blog_rss, + size: 1000, + minutes: 15 + } + } # cron-like job scheduler config :groupher_server, Helper.Scheduler, @@ -147,6 +161,11 @@ config :groupher_server, Helper.Scheduler, {"@daily", {Helper.Scheduler, :archive_artiments, []}} ] +# handle background jobs +config :rihanna, + jobs_table_name: "background_jobs", + producer_postgres_connection: {Ecto, GroupherServer.Repo} + import_config "#{Mix.env()}.exs" if File.exists?("config/#{Mix.env()}.secret.exs") do diff --git a/lib/groupher_server/application.ex b/lib/groupher_server/application.ex index ca13da300..cc6335dd1 100644 --- a/lib/groupher_server/application.ex +++ b/lib/groupher_server/application.ex @@ -1,30 +1,31 @@ defmodule GroupherServer.Application do @moduledoc false use Application + import Helper.Utils, only: [get_config: 2] + + alias Helper.Cache + + @cache_pool get_config(:cache, :pool) # See https://hexdocs.pm/elixir/Application.html # for more information on OTP Applications @spec start(any, any) :: {:error, any} | {:ok, pid} def start(_type, _args) do import Supervisor.Spec - alias Helper.Cache # Define workers and child supervisors to be supervised - children = [ - # Start the PubSub system - {Phoenix.PubSub, name: MyApp.PubSub}, - # Start the Ecto repository - supervisor(GroupherServer.Repo, []), - # Start the endpoint when the application starts - supervisor(GroupherServerWeb.Endpoint, []), - # Start your own worker by calling: GroupherServer.Worker.start_link(arg1, arg2, arg3) - # worker(GroupherServer.Worker, [arg1, arg2, arg3]), - worker(Cachex, [:common, Cache.config(:common)], id: :common), - worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login), - # - worker(Helper.Scheduler, []), - {Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]} - ] + children = + [ + # Start the PubSub system + {Phoenix.PubSub, name: MyApp.PubSub}, + # Start the Ecto repository + supervisor(GroupherServer.Repo, []), + # Start the endpoint when the application starts + supervisor(GroupherServerWeb.Endpoint, []), + # Start your own worker by calling: GroupherServer.Worker.start_link(arg1, arg2, arg3) + worker(Helper.Scheduler, []), + {Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]} + ] ++ cache_workers() # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options @@ -38,4 +39,19 @@ defmodule GroupherServer.Application do GroupherServerWeb.Endpoint.config_change(changed, removed) :ok end + + defp cache_workers() do + import Supervisor.Spec + + # worker(GroupherServer.Worker, [arg1, arg2, arg3]), + # worker(Cachex, [:common, Cache.config(:common)], id: :common), + # worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login), + # worker(Cachex, [:blog_rss, Cache.config(:blog_rss)], id: :blog_rss), + @cache_pool + |> Map.keys() + |> Enum.reduce([], fn key, acc -> + name = @cache_pool[key].name + acc ++ [worker(Cachex, [name, Cache.config(key)], id: name)] + end) + end end diff --git a/lib/groupher_server/cms/cms.ex b/lib/groupher_server/cms/cms.ex index fa5e43b99..0afe997cc 100644 --- a/lib/groupher_server/cms/cms.ex +++ b/lib/groupher_server/cms/cms.ex @@ -10,6 +10,7 @@ defmodule GroupherServer.CMS do alias Delegate.{ AbuseReport, ArticleCURD, + BlogCURD, ArticleCommunity, ArticleEmotion, CitedArtiment, @@ -102,6 +103,9 @@ defmodule GroupherServer.CMS do defdelegate archive_articles(thread), to: ArticleCURD + defdelegate create_blog_rss(attrs), to: BlogCURD + defdelegate blog_rss_feed(rss), to: BlogCURD + defdelegate paged_citing_contents(type, id, filter), to: CitedArtiment defdelegate upvote_article(thread, article_id, user), to: ArticleUpvote diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex new file mode 100644 index 000000000..1524d0898 --- /dev/null +++ b/lib/groupher_server/cms/delegates/blog_curd.ex @@ -0,0 +1,77 @@ +defmodule GroupherServer.CMS.Delegate.BlogCURD do + @moduledoc """ + CURD operation on post/job ... + """ + import Ecto.Query, warn: false + + # import Helper.Utils, only: [done: 1] + + # import Helper.ErrorCode + # import ShortMaps + + # alias Helper.{ORM} + alias GroupherServer.{CMS, Repo} + alias CMS.Model.BlogRSS + + alias Helper.{Cache, RSS} + + @cache_pool :blog_rss + + # alias Ecto.Multi + def blog_rss_feed(rss) when is_binary(rss) do + get_feed_and_cache(rss) + end + + @doc """ + get and cache user'id by user's login + """ + def get_feed_and_cache(rss) do + case Cache.get(@cache_pool, rss) do + {:ok, feed} -> {:ok, feed} + {:error, _} -> do_get_feed_and_cache(rss) + end + end + + defp do_get_feed_and_cache(rss) do + # {:ok, feed} = RSS.get(rss) + with {:ok, feed} = RSS.get(rss) do + Cache.put(@cache_pool, rss, feed) + {:ok, feed} + end + end + + def create_blog() do + # 1. 先判断 rss 是否存在 + ## 1.1 如果存在，从 cache 中获取 + ## 1.2 如不存在，则创建一条 RSS + + # 2. 创建 blog + ## 2.1 blog +字段 rss, author + ## 2.2 title, digest, xxx + + # 前台获取作者信息的时候从 rss 表读取 + end + + def create_blog_rss(attrs) do + history_feed = Map.get(attrs, :history_feed) + attrs = attrs |> Map.drop([:history_feed]) + + %BlogRSS{} + |> Ecto.Changeset.change(attrs) + |> Ecto.Changeset.put_embed(:history_feed, history_feed) + |> Repo.insert() + end + + # create done + # defp result({:ok, %{set_active_at_timestamp: result}}) do + # {:ok, result} + # end + + # defp result({:ok, %{update_article_meta: result}}), do: {:ok, result} + + # defp result({:error, :create_article, _result, _steps}) do + # {:error, [message: "create article", code: ecode(:create_fails)]} + # end + + # defp result({:error, _, result, _steps}), do: {:error, result} +end diff --git a/lib/groupher_server/cms/models/blog_rss.ex b/lib/groupher_server/cms/models/blog_rss.ex new file mode 100644 index 000000000..fdf65288d --- /dev/null +++ b/lib/groupher_server/cms/models/blog_rss.ex @@ -0,0 +1,60 @@ +defmodule GroupherServer.CMS.Model.BlogRSS do + @moduledoc false + alias __MODULE__ + + use Ecto.Schema + use Accessible + + import Ecto.Changeset + # import GroupherServer.CMS.Helper.Macros + + alias GroupherServer.CMS + alias CMS.Model.Embeds + + @timestamps_opts [type: :utc_datetime_usec] + + @required_fields ~w(link rss)a + @optional_fields ~w(subtitle author updated)a + + @type t :: %BlogRSS{} + schema "cms_blog_rss" do + field(:rss, :string) + field(:title, :string) + field(:subtitle, :string) + field(:link, :string) + field(:updated, :string) + embeds_many(:history_feed, Embeds.BlogHistoryFeed, on_replace: :delete) + embeds_one(:author, Embeds.BlogAuthor, on_replace: :update) + end + + @doc false + def changeset(%BlogRSS{} = blog_rss, attrs) do + blog_rss + |> cast(attrs, @optional_fields ++ @required_fields) + |> validate_required(@required_fields) + |> cast_embed(:history_feed, required: true, with: &Embeds.BlogHistoryFeed.changeset/2) + |> cast_embed(:author, required: false, with: &Embeds.BlogAuthor.changeset/2) + end + + @doc false + def update_changeset(%BlogRSS{} = blog_rss, attrs) do + blog_rss + |> cast(attrs, @optional_fields ++ @required_fields) + |> cast_embed(:history_feed, required: false, with: &Embeds.BlogHistoryFeed.changeset/2) + |> cast_embed(:author, required: false, with: &Embeds.BlogAuthor.changeset/2) + end + + # @doc false + # def update_changeset(%BlogRSS{} = blog_rss, attrs) do + # blog_rss + # |> cast(attrs, @optional_fields ++ @required_fields) + # |> generl_changeset + # end + + # defp generl_changeset(changeset) do + # changeset + # |> validate_length(:title, min: 3, max: 100) + # |> cast_embed(:emotions, with: &Embeds.ArticleEmotion.changeset/2) + # |> validate_length(:link_addr, min: 5, max: 400) + # end +end diff --git a/lib/groupher_server/cms/models/embeds/blog_author.ex b/lib/groupher_server/cms/models/embeds/blog_author.ex new file mode 100644 index 000000000..6bc1ed296 --- /dev/null +++ b/lib/groupher_server/cms/models/embeds/blog_author.ex @@ -0,0 +1,26 @@ +defmodule GroupherServer.CMS.Model.Embeds.BlogAuthor do + @moduledoc """ + general community meta + """ + use Ecto.Schema + use Accessible + + import Ecto.Changeset + + @required_fields ~w(name)a + @optional_fields ~w(link intro github twitter)a + + embedded_schema do + field(:name, :string) + field(:link, :string) + field(:intro, :string) + field(:github, :string) + field(:twitter, :string) + end + + def changeset(struct, attrs) do + struct + |> cast(attrs, @optional_fields ++ @required_fields) + |> validate_required(@required_fields) + end +end diff --git a/lib/groupher_server/cms/models/embeds/blog_history_feed.ex b/lib/groupher_server/cms/models/embeds/blog_history_feed.ex new file mode 100644 index 000000000..801ba71ec --- /dev/null +++ b/lib/groupher_server/cms/models/embeds/blog_history_feed.ex @@ -0,0 +1,25 @@ +defmodule GroupherServer.CMS.Model.Embeds.BlogHistoryFeed do + @moduledoc """ + general community meta + """ + use Ecto.Schema + use Accessible + + import Ecto.Changeset + + @optional_fields ~w(title digest link_addr content published_at)a + + embedded_schema do + field(:title, :string) + field(:digest, :string) + field(:link_addr, :string) + field(:content, :string) + field(:published, :string) + field(:updated, :string) + end + + def changeset(struct, params) do + struct + |> cast(params, @optional_fields) + end +end diff --git a/lib/helper/RSS.ex b/lib/helper/RSS.ex new file mode 100644 index 000000000..00f0b7895 --- /dev/null +++ b/lib/helper/RSS.ex @@ -0,0 +1,112 @@ +defmodule Helper.RSS do + @moduledoc """ + RSS get and parser + """ + def get(addr) do + with {:ok, %{body: body}} <- HTTPoison.get(addr) do + rss_parser(body) + else + error -> + IO.inspect(error, label: "error") + {:error, :invalid_rss_address} + end + end + + defp rss_parser(body) do + with {:ok, feed} <- Fiet.Atom.parse(body) do + # IO.inspect(feed, label: "atom feed") + format(:atom, feed) + else + {:error, %Fiet.Atom.ParsingError{reason: {:not_atom, "rss"}}} -> + rss_parser(body, :rss2) + end + end + + defp rss_parser(body, :rss2) do + with {:ok, feed} <- Fiet.RSS2.parse(body) do + # IO.inspect(feed, label: "rss2 feed") + format(:rss2, feed) + end + end + + defp format(:atom, %Fiet.Atom.Feed{entries: entries} = feed) do + items = + Enum.reduce(entries, [], fn item, acc -> + acc ++ [format(:item, item)] + end) + + {:ok, + %{ + title: parse(:text, feed.title), + subtitle: parse(:text, feed.subtitle), + link: parse(:link, feed), + updated: feed.updated, + history_feed: items + }} + end + + defp format(:rss2, %Fiet.RSS2.Channel{items: items} = feed) do + items = + Enum.reduce(items, [], fn item, acc -> + acc ++ [format(:item, item)] + end) + + {:ok, + %{ + title: feed.title, + subtitle: feed.description, + link: feed.link, + updated: feed.last_build_date, + history_feed: items + }} + end + + defp format(:item, %Fiet.Atom.Entry{} = item) do + %{ + title: parse(:text, item.title), + digest: parse(:digest, item), + link_addr: parse(:link, item), + # + published: parse(:published, item), + updated: item.updated + } + end + + defp format(:item, %Fiet.RSS2.Item{} = item) do + %{ + title: item.title, + digest: item.description, + link_addr: item.link, + # + published: item.pub_date, + updated: item.pub_date + } + end + + defp parse(:digest, %Fiet.Atom.Entry{summary: nil}), do: "use content TODO" + defp parse(:digest, %Fiet.Atom.Entry{summary: summary}), do: parse(:text, summary) + defp parse(:digest, _), do: "use content TODO" + + defp parse(:text, {:text, text}), do: text + defp parse(:text, _), do: "" + + defp parse(:link, %Fiet.Atom.Entry{links: links}), do: do_parse_link(links) + defp parse(:link, %Fiet.Atom.Feed{links: links}), do: do_parse_link(links) + + defp parse(:published, %Fiet.Atom.Entry{published: nil, updated: updated}) do + updated + end + + defp parse(:published, %Fiet.Atom.Entry{published: published}) do + published + end + + defp do_parse_link([]), do: "" + + defp do_parse_link(links) do + case Enum.find(links, &(&1.type === "text/html")) do + nil -> links |> List.first() |> Map.get(:href) + link -> link.href + end + end +end diff --git a/lib/helper/cache.ex b/lib/helper/cache.ex index 7f5f50857..ee5374694 100644 --- a/lib/helper/cache.ex +++ b/lib/helper/cache.ex @@ -3,26 +3,45 @@ defmodule Helper.Cache do memory cache using cachex https://github.com/whitfin/cachex """ import Cachex.Spec + import Helper.Utils, only: [get_config: 2] - def config(:common) do - [ - limit: limit(size: 5000, policy: Cachex.Policy.LRW, reclaim: 0.1), - expiration: expiration(default: :timer.minutes(10)) - ] - end + @cache_pool get_config(:cache, :pool) - @doc """ - cache config for user.login -> user.id, used in accounts resolver - user.id is a linearly increasing integer, kind sensitive, so use user.login instead - """ - def config(:user_login) do + def config(pool_name) do [ - limit: limit(size: 10_000, policy: Cachex.Policy.LRW, reclaim: 0.1), - # expired in one week, it's fine, since user's login and id will never change - expiration: expiration(default: :timer.minutes(10_080)) + limit: limit(size: @cache_pool[pool_name].size, policy: Cachex.Policy.LRW, reclaim: 0.1), + expiration: expiration(default: :timer.minutes(@cache_pool[pool_name].minutes)) ] end + # # size, minites + # def config(:common) do + # [ + # limit: limit(size: 5000, policy: Cachex.Policy.LRW, reclaim: 0.1), + # expiration: expiration(default: :timer.minutes(10)) + # ] + # end + + # @doc """ + # cache config for user.login -> user.id, used in accounts resolver + # user.id is a linearly increasing integer, kind sensitive, so use user.login instead + # """ + # def config(:user_login) do + # [ + # limit: limit(size: 10_000, policy: Cachex.Policy.LRW, reclaim: 0.1), + # # expired in one week, it's fine, since user's login and id will never change + # expiration: expiration(default: :timer.minutes(10_080)) + # ] + # end + + # def config(:blog_rss) do + # [ + # limit: limit(size: 1000, policy: Cachex.Policy.LRW, reclaim: 0.1), + # # expired in one week, it's fine, since user's login and id will never change + # expiration: expiration(default: :timer.minutes(10)) + # ] + # end + @doc """ ## Example iex> Helper.Cache.get(:common, :a) diff --git a/mix.exs b/mix.exs index 690c7708b..ecbb0136b 100644 --- a/mix.exs +++ b/mix.exs @@ -110,7 +110,10 @@ defmodule GroupherServer.Mixfile do # https://github.com/cataska/pangu.ex {:pangu, "~> 0.1.0"}, {:accessible, "~> 0.3.0"}, - {:floki, "~> 0.30.1"} + {:floki, "~> 0.30.1"}, + {:httpoison, "~> 1.8"}, + # rss feed parser + {:fiet, "~> 0.3"} ] end diff --git a/mix.lock b/mix.lock index 8ab81947b..da923df79 100644 --- a/mix.lock +++ b/mix.lock @@ -37,6 +37,7 @@ "ex_zstd": {:hex, :ex_zstd, "0.1.0", "4b1b5ebd7c0417e69308db8cdd478b9adb3e2d1a03b6e7366cf0a9aadeae11af", [:make, :mix], [{:ex_doc, ">= 0.0.0", [hex: :ex_doc, repo: "hexpm", optional: false]}], "hexpm", "2c9542a5c088e0eab14aa9b10d18bc084a6060ecf09025bbfc5b08684568bc67"}, "excoveralls": {:hex, :excoveralls, "0.14.1", "14140e4ef343f2af2de33d35268c77bc7983d7824cb945e6c2af54235bc2e61f", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "4a588f9f8cf9dc140cc1f3d0ea4d849b2f76d5d8bee66b73c304bb3d3689c8b0"}, "faker": {:hex, :faker, "0.16.0", "1e2cf3e8d60d44a30741fb98118fcac18b2020379c7e00d18f1a005841b2f647", [:mix], [], "hexpm", "fbcb9bf1299dff3c9dd7e50f41802bbc472ffbb84e7656394c8aa913ec315141"}, + "fiet": {:hex, :fiet, "0.3.0", "fdfc03119250e7f4b2eac88a03325d43af80b08e55aa80352d0d138a972815f9", [:mix], [{:saxy, "~> 1.2", [hex: :saxy, repo: "hexpm", optional: false]}], "hexpm", "b5cff4338b0b216c04b489dd99c839524469d7d21cd6dc9717768497276810eb"}, "file_system": {:hex, :file_system, "0.2.8", "f632bd287927a1eed2b718f22af727c5aeaccc9a98d8c2bd7bff709e851dc986", [:mix], [], "hexpm", "97a3b6f8d63ef53bd0113070102db2ce05352ecf0d25390eb8d747c2bde98bca"}, "floki": {:hex, :floki, "0.30.1", "75d35526d3a1459920b6e87fdbc2e0b8a3670f965dd0903708d2b267e0904c55", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e9c03524447d1c4cbfccd672d739b8c18453eee377846b119d4fd71b1a176bb8"}, "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"}, @@ -47,7 +48,7 @@ "hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.1", "e8a67da405fe9f0d1be121a40a60f70811192033a5b8d00a95dddd807f5e053e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "68d92656f47cd73598c45ad2394561f025c8c65d146001b955fd7b517858962a"}, - "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, + "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inch_ex": {:hex, :inch_ex, "2.0.0", "24268a9284a1751f2ceda569cd978e1fa394c977c45c331bb52a405de544f4de", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "96d0ec5ecac8cf63142d02f16b7ab7152cf0f0f1a185a80161b758383c9399a8"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, @@ -86,6 +87,7 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, "recase": {:hex, :recase, "0.7.0", "3f2f719f0886c7a3b7fe469058ec539cb7bbe0023604ae3bce920e186305e5ae", [:mix], [], "hexpm", "36f5756a9f552f4a94b54a695870e32f4e72d5fad9c25e61bc4a3151c08a4e0c"}, "rihanna": {:hex, :rihanna, "1.3.5", "5f5e6c5b1e514978a29a6791f338f4bb963401959fc212bd18d4a2c92d79a7a4", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.13.3", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "fa1918c2ab63c8ada9a23ad6fe03cd181378739f0ff10741b45d0bcb50003c74"}, + "saxy": {:hex, :saxy, "1.4.0", "c7203ad20001f72eaaad07d08f82be063fa94a40924e6bb39d93d55f979abcba", [:mix], [], "hexpm", "3fe790354d3f2234ad0b5be2d99822a23fa2d4e8ccd6657c672901dac172e9a9"}, "scrivener": {:hex, :scrivener, "2.5.0", "e1f78c62b6806d91cc9c4778deef1ea4e80aa9fadfce2c16831afe0468cc8a2c", [:mix], [], "hexpm", "c3e484da6bb7084b5a24c7e38a8ca09310d5fbf5241db05f625fb8af557ef667"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, "sentry": {:hex, :sentry, "7.1.0", "546729ea0be4a3f593b456fe77a2cf5537e390fbe87c191424557dae8c2bd760", [:mix], [{:hackney, "~> 1.8 or 1.6.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "6e439c2b93a3d1e049eeaa7910d9ef4ff255b107b2192c25fe50e5f2cd57d2d3"}, diff --git a/priv/repo/migrations/20210924023609_create_blog_rss.exs b/priv/repo/migrations/20210924023609_create_blog_rss.exs new file mode 100644 index 000000000..c112c445a --- /dev/null +++ b/priv/repo/migrations/20210924023609_create_blog_rss.exs @@ -0,0 +1,17 @@ +defmodule GroupherServer.Repo.Migrations.CreateBlogRss do + use Ecto.Migration + + def change do + create table(:cms_blog_rss) do + add(:rss, :string) + add(:link, :string) + add(:title, :string) + add(:subtitle, :string) + add(:updated, :string) + add(:history_feed, :map) + add(:author, :map) + end + + create(unique_index(:cms_blog_rss, [:rss, :link])) + end +end diff --git a/test/groupher_server_web/query/cms/comments/guide_comment_test.exs b/test/groupher_server_web/query/cms/comments/guide_comment_test.exs index 5974ccf64..c21568777 100644 --- a/test/groupher_server_web/query/cms/comments/guide_comment_test.exs +++ b/test/groupher_server_web/query/cms/comments/guide_comment_test.exs @@ -30,7 +30,6 @@ defmodule GroupherServer.Test.Query.Comments.GuideComment do } } """ - @tag :wip test "guest user can get comment participants after comment created", ~m(guest_conn guide user user2)a do total_count = 5 diff --git a/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs b/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs index 439482bb3..7fbe74a75 100644 --- a/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs +++ b/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs @@ -193,7 +193,6 @@ defmodule GroupherServer.Test.Query.Comments.MeetupComment do assert random_comment["repliesCount"] == 2 end - @tag :wip test "comment should have reply_to content if need", ~m(guest_conn meetup user user2)a do total_count = 2 thread = :meetup diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs new file mode 100644 index 000000000..e26283a34 --- /dev/null +++ b/test/helper/rss_test.exs @@ -0,0 +1,41 @@ +defmodule GroupherServer.Test.Helper.RSSTest do + use GroupherServer.TestTools + + alias GroupherServer.CMS + alias Helper.{Cache} + + @cache_pool :blog_rss + + describe "get rss" do + @tag :wip + test "parse and create basic rss." do + rss = mock_rss_addr() + {:ok, feed} = CMS.blog_rss_feed(rss) + feed = feed |> Map.merge(%{rss: "rss-addr, todo"}) + + {:ok, rss_record} = CMS.create_blog_rss(feed) + assert rss_record.history_feed |> length !== 0 + + {:ok, cache} = Cache.get(@cache_pool, rss) + assert not is_nil(cache) + end + + @tag :wip + test "create rss with author" do + {:ok, feed} = CMS.blog_rss_feed(mock_rss_addr()) + + author = %{ + name: "mydearxym", + link: "https://coderplaents.com" + } + + feed = + feed + |> Map.merge(%{rss: "rss-addr, todo"}) + |> Map.merge(%{author: author}) + + {:ok, rss_record} = CMS.create_blog_rss(feed) + assert rss_record.author.name == "mydearxym" + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index e4ead7c9a..c6ed4f426 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -539,6 +539,15 @@ defmodule GroupherServer.Support.Factory do @images |> Enum.slice(0, count) end + def mock_rss_addr() do + # "https://www.xiabingbao.com/atom.xml" # 不规范 + # "https://rsshub.app/blogs/wangyin" + "https://www.zhangxinxu.com/wordpress/feed/" + # "https://overreacted.io/rss.xml" + # "https://www.ruanyifeng.com/blog/atom.xml" + # "https://lutaonan.com/rss.xml" + end + def mock_mention_for(user, from_user) do {:ok, post} = db_insert(:post) From fba92d5138d2220c6c72898a07e21927e18eed37 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Sat, 25 Sep 2021 17:55:31 +0800 Subject: [PATCH 2/6] refactor(rss): blog create logic wip --- lib/groupher_server/cms/cms.ex | 2 + .../cms/delegates/blog_curd.ex | 80 ++++++++++++++----- lib/helper/RSS.ex | 9 ++- test/helper/rss_test.exs | 55 ++++++++++--- 4 files changed, 113 insertions(+), 33 deletions(-) diff --git a/lib/groupher_server/cms/cms.ex b/lib/groupher_server/cms/cms.ex index 0afe997cc..12d34f718 100644 --- a/lib/groupher_server/cms/cms.ex +++ b/lib/groupher_server/cms/cms.ex @@ -103,7 +103,9 @@ defmodule GroupherServer.CMS do defdelegate archive_articles(thread), to: ArticleCURD + defdelegate create_blog(community, attrs, user), to: BlogCURD defdelegate create_blog_rss(attrs), to: BlogCURD + defdelegate update_blog_rss(attrs), to: BlogCURD defdelegate blog_rss_feed(rss), to: BlogCURD defdelegate paged_citing_contents(type, id, filter), to: CitedArtiment diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex index 1524d0898..3c1a42ec6 100644 --- a/lib/groupher_server/cms/delegates/blog_curd.ex +++ b/lib/groupher_server/cms/delegates/blog_curd.ex @@ -3,6 +3,7 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do CURD operation on post/job ... """ import Ecto.Query, warn: false + import Helper.Utils, only: [strip_struct: 1] # import Helper.Utils, only: [done: 1] @@ -10,40 +11,37 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # import ShortMaps # alias Helper.{ORM} - alias GroupherServer.{CMS, Repo} - alias CMS.Model.BlogRSS + alias GroupherServer.{Accounts, CMS, Repo} + alias CMS.Model.{BlogRSS, Community} + alias Accounts.Model.User - alias Helper.{Cache, RSS} + alias Helper.{ORM, Cache, RSS} @cache_pool :blog_rss # alias Ecto.Multi def blog_rss_feed(rss) when is_binary(rss) do - get_feed_and_cache(rss) - end - - @doc """ - get and cache user'id by user's login - """ - def get_feed_and_cache(rss) do - case Cache.get(@cache_pool, rss) do - {:ok, feed} -> {:ok, feed} - {:error, _} -> do_get_feed_and_cache(rss) - end - end - - defp do_get_feed_and_cache(rss) do - # {:ok, feed} = RSS.get(rss) - with {:ok, feed} = RSS.get(rss) do - Cache.put(@cache_pool, rss, feed) + with {:ok, feed} <- ORM.find_by(BlogRSS, %{rss: rss}) do {:ok, feed} + else + _ -> fetch_fresh_feed_and_cache(rss) end end - def create_blog() do + # attrs 包含 rss, blog_title + # def create_article(%Community{id: cid}, thread, attrs, %User{id: uid}) do + def create_blog(%Community{} = community, attrs, %User{} = user) do # 1. 先判断 rss 是否存在 ## 1.1 如果存在，从 cache 中获取 ## 1.2 如不存在，则创建一条 RSS + with {:ok, feed} <- blog_rss_feed(attrs.rss) do + do_create_blog(community, attrs, user, feed) + + # IO.inspect(feed, label: "create blog") + # 通过 feed 有没有 id 来 insert / update + # 通过 blog_title, 组合 attrs 传给 create_article + {:ok, :pass} + end # 2. 创建 blog ## 2.1 blog +字段 rss, author @@ -52,6 +50,14 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # 前台获取作者信息的时候从 rss 表读取 end + defp do_create_blog(%Community{id: cid}, attrs, %User{id: uid}, %{id: id} = feed) do + IO.inspect("rss 记录存在, 直接创建 blog", label: "do_create_blog") + end + + defp do_create_blog(%Community{id: cid}, attrs, %User{id: uid}, feed) do + IO.inspect("rss 记录不存在, 先创建 rss, 再创建 blog", label: "do_create_blog") + end + def create_blog_rss(attrs) do history_feed = Map.get(attrs, :history_feed) attrs = attrs |> Map.drop([:history_feed]) @@ -62,6 +68,20 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do |> Repo.insert() end + def update_blog_rss(%{rss: rss} = attrs) do + with {:ok, blog_rss} <- ORM.find_by(BlogRSS, rss: rss) do + history_feed = + Map.get(attrs, :history_feed, Enum.map(blog_rss.history_feed, &strip_struct(&1))) + + attrs = attrs |> Map.drop([:history_feed]) + + %BlogRSS{} + |> Ecto.Changeset.change(attrs) + |> Ecto.Changeset.put_embed(:history_feed, history_feed) + |> Repo.insert() + end + end + # create done # defp result({:ok, %{set_active_at_timestamp: result}}) do # {:ok, result} @@ -74,4 +94,22 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # end # defp result({:error, _, result, _steps}), do: {:error, result} + + @doc """ + get and cache feed by rss address as key + """ + def fetch_fresh_feed_and_cache(rss) do + case Cache.get(@cache_pool, rss) do + {:ok, feed} -> {:ok, feed} + {:error, _} -> get_feed_and_cache(rss) + end + end + + defp get_feed_and_cache(rss) do + # {:ok, feed} = RSS.get(rss) + with {:ok, feed} = RSS.get(rss) do + Cache.put(@cache_pool, rss, feed) + {:ok, feed} + end + end end diff --git a/lib/helper/RSS.ex b/lib/helper/RSS.ex index 00f0b7895..c0f45dc99 100644 --- a/lib/helper/RSS.ex +++ b/lib/helper/RSS.ex @@ -2,9 +2,12 @@ defmodule Helper.RSS do @moduledoc """ RSS get and parser """ - def get(addr) do - with {:ok, %{body: body}} <- HTTPoison.get(addr) do - rss_parser(body) + import Helper.Utils, only: [done: 1] + + def get(rss) do + with {:ok, %{body: body}} <- HTTPoison.get(rss), + {:ok, blog_rss} <- rss_parser(body) do + blog_rss |> Map.merge(%{rss: rss}) |> done else error -> IO.inspect(error, label: "error") diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs index e26283a34..7cda1ecc5 100644 --- a/test/helper/rss_test.exs +++ b/test/helper/rss_test.exs @@ -1,28 +1,50 @@ defmodule GroupherServer.Test.Helper.RSSTest do + @moduledoc false use GroupherServer.TestTools alias GroupherServer.CMS alias Helper.{Cache} @cache_pool :blog_rss + @rss mock_rss_addr() - describe "get rss" do + setup do + {:ok, community} = db_insert(:community) + {:ok, user} = db_insert(:user) + blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + + {:ok, ~m(community user blog_attrs)a} + end + + describe "blog curd" do @tag :wip - test "parse and create basic rss." do - rss = mock_rss_addr() - {:ok, feed} = CMS.blog_rss_feed(rss) - feed = feed |> Map.merge(%{rss: "rss-addr, todo"}) + test "can create blog", ~m(community user blog_attrs)a do + {:ok, feed} = CMS.blog_rss_feed(@rss) + {:ok, _rss_record} = CMS.create_blog_rss(feed) + + blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + + blog_attrs = %{rss: @rss} + {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + end + end + + describe "fetch rss & curd" do + @tag :wip2 + test "parse and create basic rss" do + {:ok, feed} = CMS.blog_rss_feed(@rss) + feed = feed |> Map.merge(%{rss: @rss}) {:ok, rss_record} = CMS.create_blog_rss(feed) assert rss_record.history_feed |> length !== 0 - {:ok, cache} = Cache.get(@cache_pool, rss) + {:ok, cache} = Cache.get(@cache_pool, @rss) assert not is_nil(cache) end - @tag :wip + @tag :wip2 test "create rss with author" do - {:ok, feed} = CMS.blog_rss_feed(mock_rss_addr()) + {:ok, feed} = CMS.blog_rss_feed(@rss) author = %{ name: "mydearxym", @@ -31,11 +53,26 @@ defmodule GroupherServer.Test.Helper.RSSTest do feed = feed - |> Map.merge(%{rss: "rss-addr, todo"}) + |> Map.merge(%{rss: @rss}) |> Map.merge(%{author: author}) {:ok, rss_record} = CMS.create_blog_rss(feed) assert rss_record.author.name == "mydearxym" end + + @tag :wip2 + test "update rss with author and exsit feed" do + {:ok, feed} = CMS.blog_rss_feed(@rss) + {:ok, rss_record} = CMS.create_blog_rss(feed) + + author = %{ + name: "mydearxym", + link: "https://coderplaents.com" + } + + attrs = %{rss: rss_record.rss, author: author, history_feed: rss_record.history_feed} + {:ok, rss_record} = CMS.update_blog_rss(attrs) + assert rss_record.author.name == "mydearxym" + end end end From bfdaae4a7cf250206b6bf7fc55d1d8a2878aa275 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Sat, 25 Sep 2021 23:29:13 +0800 Subject: [PATCH 3/6] refactor(rss): blog create logic wip --- .../cms/delegates/blog_curd.ex | 22 +++++++++++++++++-- .../cms/article_tags/post_tag_test.exs | 2 +- test/groupher_server/seeds/clean_up_test.exs | 2 +- test/helper/rss_test.exs | 14 ++++++++++-- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex index 3c1a42ec6..a667a09e0 100644 --- a/lib/groupher_server/cms/delegates/blog_curd.ex +++ b/lib/groupher_server/cms/delegates/blog_curd.ex @@ -5,6 +5,7 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do import Ecto.Query, warn: false import Helper.Utils, only: [strip_struct: 1] + import GroupherServer.CMS.Delegate.ArticleCURD, only: [create_article: 4] # import Helper.Utils, only: [done: 1] # import Helper.ErrorCode @@ -40,7 +41,6 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # IO.inspect(feed, label: "create blog") # 通过 feed 有没有 id 来 insert / update # 通过 blog_title, 组合 attrs 传给 create_article - {:ok, :pass} end # 2. 创建 blog @@ -50,12 +50,30 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # 前台获取作者信息的时候从 rss 表读取 end - defp do_create_blog(%Community{id: cid}, attrs, %User{id: uid}, %{id: id} = feed) do + defp do_create_blog(%Community{} = community, attrs, %User{} = user, %{id: _} = feed) do IO.inspect("rss 记录存在, 直接创建 blog", label: "do_create_blog") + + # author = feed.author + selected_feed = Enum.find(feed.history_feed, &(&1.title == attrs.title)) + IO.inspect(selected_feed, label: "target feed") + IO.inspect(feed, label: "the author") + # IO.inspect(feed, label: "feed -") + + attrs = + attrs + |> Map.merge(%{link_addr: selected_feed.link_addr, published: selected_feed.published}) + + IO.inspect(attrs, label: "attrs -") + create_article(community, :blog, attrs, user) + # arg(:title, non_null(:string)) + # arg(:body, non_null(:string)) + # arg(:community_id, non_null(:id)) + # arg(:link_addr, :string) end defp do_create_blog(%Community{id: cid}, attrs, %User{id: uid}, feed) do IO.inspect("rss 记录不存在, 先创建 rss, 再创建 blog", label: "do_create_blog") + {:ok, :pass} end def create_blog_rss(attrs) do diff --git a/test/groupher_server/cms/article_tags/post_tag_test.exs b/test/groupher_server/cms/article_tags/post_tag_test.exs index a69b6186b..fdf45468d 100644 --- a/test/groupher_server/cms/article_tags/post_tag_test.exs +++ b/test/groupher_server/cms/article_tags/post_tag_test.exs @@ -139,7 +139,7 @@ defmodule GroupherServer.Test.CMS.ArticleTag.PostTag do assert not exist_in?(article_tag2, post.article_tags) end - test "can not set dup tag ", ~m(community post article_tag_attrs article_tag_attrs2 user)a do + test "can not set dup tag ", ~m(community post article_tag_attrs user)a do {:ok, article_tag} = CMS.create_article_tag(community, :post, article_tag_attrs, user) {:ok, post} = CMS.set_article_tag(:post, post.id, article_tag.id) {:ok, post} = CMS.set_article_tag(:post, post.id, article_tag.id) diff --git a/test/groupher_server/seeds/clean_up_test.exs b/test/groupher_server/seeds/clean_up_test.exs index 527cafe40..af6d9c2d4 100644 --- a/test/groupher_server/seeds/clean_up_test.exs +++ b/test/groupher_server/seeds/clean_up_test.exs @@ -22,7 +22,7 @@ defmodule GroupherServer.Test.Seeds.CleanUp do describe "[community clean up]" do test "can clean up a community", ~m(user post_attrs)a do {:ok, community} = CMS.seed_community(:home) - {:ok, post} = CMS.create_article(community, :post, post_attrs, user) + {:ok, _post} = CMS.create_article(community, :post, post_attrs, user) {:ok, found} = ORM.find_all(ArticleTag, %{page: 1, size: 20}) assert found.total_count !== 0 diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs index 7cda1ecc5..60ebc8c30 100644 --- a/test/helper/rss_test.exs +++ b/test/helper/rss_test.exs @@ -22,10 +22,20 @@ defmodule GroupherServer.Test.Helper.RSSTest do {:ok, feed} = CMS.blog_rss_feed(@rss) {:ok, _rss_record} = CMS.create_blog_rss(feed) - blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + selected_feed = feed.history_feed |> List.first() + title = selected_feed |> Map.get(:title) + link_addr = selected_feed |> Map.get(:link_addr) + # blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + blog_attrs = %{ + rss: @rss, + title: title, + body: mock_rich_text("pleace use content field instead") + } - blog_attrs = %{rss: @rss} {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + assert blog.title == title + assert blog.link_addr == link_addr + # blog end end From 73e65cc0829a78327e2bb24e3c62e5e49b09cd25 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Sun, 26 Sep 2021 12:38:57 +0800 Subject: [PATCH 4/6] refactor(rss): blog create logic wip --- .../cms/delegates/blog_curd.ex | 31 +++++----- lib/groupher_server/cms/models/blog.ex | 9 ++- ...20210926015205_add_feed_fields_to_blog.exs | 12 ++++ test/helper/rss_test.exs | 59 +++++++++++++++++-- 4 files changed, 91 insertions(+), 20 deletions(-) create mode 100644 priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex index a667a09e0..c325f7abd 100644 --- a/lib/groupher_server/cms/delegates/blog_curd.ex +++ b/lib/groupher_server/cms/delegates/blog_curd.ex @@ -50,30 +50,31 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # 前台获取作者信息的时候从 rss 表读取 end + # rss 记录存在, 直接创建 blog defp do_create_blog(%Community{} = community, attrs, %User{} = user, %{id: _} = feed) do - IO.inspect("rss 记录存在, 直接创建 blog", label: "do_create_blog") - - # author = feed.author + blog_author = if is_nil(feed.author), do: nil, else: Map.from_struct(feed.author) selected_feed = Enum.find(feed.history_feed, &(&1.title == attrs.title)) - IO.inspect(selected_feed, label: "target feed") - IO.inspect(feed, label: "the author") - # IO.inspect(feed, label: "feed -") + # TODO: feed_digest, feed_content attrs = attrs - |> Map.merge(%{link_addr: selected_feed.link_addr, published: selected_feed.published}) + |> Map.merge(%{ + link_addr: selected_feed.link_addr, + published: selected_feed.published, + blog_author: blog_author + }) + |> Enum.reject(fn {_, v} -> is_nil(v) end) + |> Map.new() - IO.inspect(attrs, label: "attrs -") create_article(community, :blog, attrs, user) - # arg(:title, non_null(:string)) - # arg(:body, non_null(:string)) - # arg(:community_id, non_null(:id)) - # arg(:link_addr, :string) end - defp do_create_blog(%Community{id: cid}, attrs, %User{id: uid}, feed) do - IO.inspect("rss 记录不存在, 先创建 rss, 再创建 blog", label: "do_create_blog") - {:ok, :pass} + # rss 记录不存在, 先创建 rss, 再创建 blog + defp do_create_blog(%Community{} = community, attrs, %User{} = user, feed) do + with {:ok, feed} <- CMS.blog_rss_feed(attrs.rss), + {:ok, feed} <- create_blog_rss(feed) do + do_create_blog(community, attrs, user, feed) + end end def create_blog_rss(attrs) do diff --git a/lib/groupher_server/cms/models/blog.ex b/lib/groupher_server/cms/models/blog.ex index 032ef3930..225333f24 100644 --- a/lib/groupher_server/cms/models/blog.ex +++ b/lib/groupher_server/cms/models/blog.ex @@ -15,13 +15,18 @@ defmodule GroupherServer.CMS.Model.Blog do @required_fields ~w(title digest)a @article_cast_fields general_article_cast_fields() - @optional_fields ~w(digest)a ++ @article_cast_fields + @optional_fields ~w(digest feed_digest feed_content published)a ++ @article_cast_fields @type t :: %Blog{} schema "cms_blogs" do # for frontend constant field(:copy_right, :string, default: "", virtual: true) + field(:feed_digest, :string) + field(:feed_content, :string) + field(:published, :string) + embeds_one(:blog_author, Embeds.BlogAuthor, on_replace: :update) + article_tags_field(:blog) article_communities_field(:blog) general_article_fields(:blog) @@ -33,6 +38,7 @@ defmodule GroupherServer.CMS.Model.Blog do |> cast(attrs, @optional_fields ++ @required_fields) |> validate_required(@required_fields) |> cast_embed(:meta, required: false, with: &Embeds.ArticleMeta.changeset/2) + |> cast_embed(:blog_author, required: false, with: &Embeds.BlogAuthor.changeset/2) |> generl_changeset end @@ -40,6 +46,7 @@ defmodule GroupherServer.CMS.Model.Blog do def update_changeset(%Blog{} = blog, attrs) do blog |> cast(attrs, @optional_fields ++ @required_fields) + |> cast_embed(:blog_author, required: false, with: &Embeds.BlogAuthor.changeset/2) |> generl_changeset end diff --git a/priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs b/priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs new file mode 100644 index 000000000..a049c957a --- /dev/null +++ b/priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs @@ -0,0 +1,12 @@ +defmodule GroupherServer.Repo.Migrations.AddFeedFieldsToBlog do + use Ecto.Migration + + def change do + alter table(:cms_blogs) do + add(:feed_digest, :string) + add(:feed_content, :text) + add(:published, :string) + add(:blog_author, :map) + end + end +end diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs index 60ebc8c30..edb78f4b6 100644 --- a/test/helper/rss_test.exs +++ b/test/helper/rss_test.exs @@ -35,12 +35,63 @@ defmodule GroupherServer.Test.Helper.RSSTest do {:ok, blog} = CMS.create_blog(community, blog_attrs, user) assert blog.title == title assert blog.link_addr == link_addr - # blog + end + + @tag :wip + test "can create blog with no-exsit rss record", ~m(community user blog_attrs)a do + {:ok, feed} = CMS.blog_rss_feed(@rss) + + selected_feed = feed.history_feed |> List.first() + title = selected_feed |> Map.get(:title) + link_addr = selected_feed |> Map.get(:link_addr) + # blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + blog_attrs = %{ + rss: @rss, + title: title, + body: mock_rich_text("pleace use content field instead") + } + + {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + assert blog.title == title + assert blog.link_addr == link_addr + end + + @tag :wip + test "can create blog with blog_author", ~m(community user blog_attrs)a do + {:ok, feed} = CMS.blog_rss_feed(@rss) + + author = %{ + name: "mydearxym", + link: "https://coderplaents.com" + } + + feed = + feed + |> Map.merge(%{rss: @rss}) + |> Map.merge(%{author: author}) + + {:ok, _rss_record} = CMS.create_blog_rss(feed) + + selected_feed = feed.history_feed |> List.first() + title = selected_feed |> Map.get(:title) + link_addr = selected_feed |> Map.get(:link_addr) + # blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + blog_attrs = %{ + rss: @rss, + title: title, + body: mock_rich_text("pleace use content field instead") + } + + {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + assert blog.title == title + assert blog.link_addr == link_addr + assert blog.blog_author.name == author.name + assert blog.blog_author.link == author.link end end describe "fetch rss & curd" do - @tag :wip2 + @tag :wip test "parse and create basic rss" do {:ok, feed} = CMS.blog_rss_feed(@rss) feed = feed |> Map.merge(%{rss: @rss}) @@ -52,7 +103,7 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert not is_nil(cache) end - @tag :wip2 + @tag :wip test "create rss with author" do {:ok, feed} = CMS.blog_rss_feed(@rss) @@ -70,7 +121,7 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert rss_record.author.name == "mydearxym" end - @tag :wip2 + @tag :wip test "update rss with author and exsit feed" do {:ok, feed} = CMS.blog_rss_feed(@rss) {:ok, rss_record} = CMS.create_blog_rss(feed) From 7b0deec5bfb013e81a815462fbdaf7e353d48f8c Mon Sep 17 00:00:00 2001 From: mydearxym Date: Wed, 29 Sep 2021 13:48:15 +0800 Subject: [PATCH 5/6] chore(blog_rss): naming & error handling --- lib/groupher_server/cms/cms.ex | 2 +- .../cms/delegates/blog_curd.ex | 27 +++++----- .../resolvers/cms_resolver.ex | 5 ++ .../schema/cms/cms_queries.ex | 7 +++ .../schema/cms/cms_types.ex | 26 ++++++++++ lib/helper/error_code.ex | 1 + .../query/cms/blog_rss_test.exs | 52 +++++++++++++++++++ test/helper/rss_test.exs | 35 +++++++------ 8 files changed, 125 insertions(+), 30 deletions(-) create mode 100644 test/groupher_server_web/query/cms/blog_rss_test.exs diff --git a/lib/groupher_server/cms/cms.ex b/lib/groupher_server/cms/cms.ex index 12d34f718..be69a21b5 100644 --- a/lib/groupher_server/cms/cms.ex +++ b/lib/groupher_server/cms/cms.ex @@ -106,7 +106,7 @@ defmodule GroupherServer.CMS do defdelegate create_blog(community, attrs, user), to: BlogCURD defdelegate create_blog_rss(attrs), to: BlogCURD defdelegate update_blog_rss(attrs), to: BlogCURD - defdelegate blog_rss_feed(rss), to: BlogCURD + defdelegate blog_rss_info(rss), to: BlogCURD defdelegate paged_citing_contents(type, id, filter), to: CitedArtiment diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex index c325f7abd..beeea75a0 100644 --- a/lib/groupher_server/cms/delegates/blog_curd.ex +++ b/lib/groupher_server/cms/delegates/blog_curd.ex @@ -3,7 +3,8 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do CURD operation on post/job ... """ import Ecto.Query, warn: false - import Helper.Utils, only: [strip_struct: 1] + import Helper.Utils, only: [strip_struct: 1, done: 1] + import Helper.ErrorCode import GroupherServer.CMS.Delegate.ArticleCURD, only: [create_article: 4] # import Helper.Utils, only: [done: 1] @@ -21,11 +22,11 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do @cache_pool :blog_rss # alias Ecto.Multi - def blog_rss_feed(rss) when is_binary(rss) do + def blog_rss_info(rss) when is_binary(rss) do with {:ok, feed} <- ORM.find_by(BlogRSS, %{rss: rss}) do {:ok, feed} else - _ -> fetch_fresh_feed_and_cache(rss) + _ -> fetch_fresh_rssinfo_and_cache(rss) end end @@ -35,7 +36,7 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # 1. 先判断 rss 是否存在 ## 1.1 如果存在，从 cache 中获取 ## 1.2 如不存在，则创建一条 RSS - with {:ok, feed} <- blog_rss_feed(attrs.rss) do + with {:ok, feed} <- blog_rss_info(attrs.rss) do do_create_blog(community, attrs, user, feed) # IO.inspect(feed, label: "create blog") @@ -71,7 +72,7 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do # rss 记录不存在, 先创建 rss, 再创建 blog defp do_create_blog(%Community{} = community, attrs, %User{} = user, feed) do - with {:ok, feed} <- CMS.blog_rss_feed(attrs.rss), + with {:ok, feed} <- CMS.blog_rss_info(attrs.rss), {:ok, feed} <- create_blog_rss(feed) do do_create_blog(community, attrs, user, feed) end @@ -117,18 +118,20 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do @doc """ get and cache feed by rss address as key """ - def fetch_fresh_feed_and_cache(rss) do + def fetch_fresh_rssinfo_and_cache(rss) do case Cache.get(@cache_pool, rss) do - {:ok, feed} -> {:ok, feed} - {:error, _} -> get_feed_and_cache(rss) + {:ok, rssinfo} -> {:ok, rssinfo} + {:error, _} -> get_rssinfo_and_cache(rss) end end - defp get_feed_and_cache(rss) do + defp get_rssinfo_and_cache(rss) do # {:ok, feed} = RSS.get(rss) - with {:ok, feed} = RSS.get(rss) do - Cache.put(@cache_pool, rss, feed) - {:ok, feed} + with {:ok, rssinfo} <- RSS.get(rss) do + Cache.put(@cache_pool, rss, rssinfo) + {:ok, rssinfo} + else + {:error, _} -> {:error, [message: "blog rss is invalid", code: ecode(:invalid_blog_rss)]} end end end diff --git a/lib/groupher_server_web/resolvers/cms_resolver.ex b/lib/groupher_server_web/resolvers/cms_resolver.ex index b29c72c39..58be1c35c 100644 --- a/lib/groupher_server_web/resolvers/cms_resolver.ex +++ b/lib/groupher_server_web/resolvers/cms_resolver.ex @@ -62,6 +62,11 @@ defmodule GroupherServerWeb.Resolvers.CMS do CMS.paged_reports(filter) end + # TODO: login only + def blog_rss_info(_root, ~m(rss)a, _) do + CMS.blog_rss_info(rss) + end + def wiki(_root, ~m(community)a, _info), do: CMS.get_wiki(%Community{raw: community}) def cheatsheet(_root, ~m(community)a, _info), do: CMS.get_cheatsheet(%Community{raw: community}) diff --git a/lib/groupher_server_web/schema/cms/cms_queries.ex b/lib/groupher_server_web/schema/cms/cms_queries.ex index 6ed0a3284..33b1b6baf 100644 --- a/lib/groupher_server_web/schema/cms/cms_queries.ex +++ b/lib/groupher_server_web/schema/cms/cms_queries.ex @@ -140,6 +140,13 @@ defmodule GroupherServerWeb.Schema.CMS.Queries do resolve(&R.CMS.search_communities/3) end + @desc "get rss info based on blog rss address" + field :blog_rss_info, :blog_rss do + arg(:rss, non_null(:string)) + + resolve(&R.CMS.blog_rss_info/3) + end + article_search_queries() article_reacted_users_query(:upvot, &R.CMS.upvoted_users/3) diff --git a/lib/groupher_server_web/schema/cms/cms_types.ex b/lib/groupher_server_web/schema/cms/cms_types.ex index adb0ade7f..202bc3c33 100644 --- a/lib/groupher_server_web/schema/cms/cms_types.ex +++ b/lib/groupher_server_web/schema/cms/cms_types.ex @@ -347,6 +347,32 @@ defmodule GroupherServerWeb.Schema.CMS.Types do timestamp_fields() end + object :blog_feed do + field(:title, :string) + field(:digest, :string) + field(:link_addr, :string) + field(:content, :string) + field(:published, :string) + field(:updated, :string) + end + + object :blog_author do + field(:name, :string) + field(:intro, :string) + field(:github, :string) + field(:twitter, :string) + end + + object :blog_rss do + field(:rss, :string) + field(:title, :string) + field(:subtitle, :string) + field(:link, :string) + field(:updated, :string) + field(:author, :blog_author) + field(:history_feed, list_of(:blog_feed)) + end + paged_article_objects() object :paged_reports do diff --git a/lib/helper/error_code.ex b/lib/helper/error_code.ex index 6be1a230d..e947c7c16 100644 --- a/lib/helper/error_code.ex +++ b/lib/helper/error_code.ex @@ -53,6 +53,7 @@ defmodule Helper.ErrorCode do def ecode(:require_questioner), do: @article_base + 9 def ecode(:cite_artilce), do: @article_base + 10 def ecode(:archived), do: @article_base + 11 + def ecode(:invalid_blog_rss), do: @article_base + 12 # def ecode(:already_solved), do: @article_base + 10 def ecode, do: @default_base diff --git a/test/groupher_server_web/query/cms/blog_rss_test.exs b/test/groupher_server_web/query/cms/blog_rss_test.exs new file mode 100644 index 000000000..95c81f8be --- /dev/null +++ b/test/groupher_server_web/query/cms/blog_rss_test.exs @@ -0,0 +1,52 @@ +defmodule GroupherServer.Test.Query.CMS.BlogRSS do + use GroupherServer.TestTools + + @rss mock_rss_addr() + + setup do + guest_conn = simu_conn(:guest) + user_conn = simu_conn(:user) + + {:ok, ~m(user_conn guest_conn)a} + end + + @query """ + query($rss: String!) { + blogRssInfo(rss: $rss) { + title + subtitle + link + updated + author { + name + intro + github + twitter + } + historyFeed { + title + digest + linkAddr + content + published + updated + } + } + } + """ + # @tag :wip2 + test "basic graphql query blog rss info", ~m(user_conn)a do + variables = %{rss: @rss} + results = user_conn |> query_result(@query, variables, "blogRssInfo") + + assert not is_nil(results["title"]) + end + + @tag :wip + test "invalid rss will get error", ~m(user_conn)a do + variables = %{rss: "invalid rss address"} + # results = user_conn |> query_result(@query, variables, "blogRssInfo") + assert user_conn |> query_get_error?(@query, variables, ecode(:invalid_blog_rss)) + # IO.inspect(results, label: "iiii") + end +end diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs index edb78f4b6..2aa9ddd4a 100644 --- a/test/helper/rss_test.exs +++ b/test/helper/rss_test.exs @@ -11,15 +11,14 @@ defmodule GroupherServer.Test.Helper.RSSTest do setup do {:ok, community} = db_insert(:community) {:ok, user} = db_insert(:user) - blog_attrs = mock_attrs(:blog, %{community_id: community.id}) - {:ok, ~m(community user blog_attrs)a} + {:ok, ~m(community user)a} end describe "blog curd" do - @tag :wip - test "can create blog", ~m(community user blog_attrs)a do - {:ok, feed} = CMS.blog_rss_feed(@rss) + @tag :wip2 + test "can create blog", ~m(community user)a do + {:ok, feed} = CMS.blog_rss_info(@rss) {:ok, _rss_record} = CMS.create_blog_rss(feed) selected_feed = feed.history_feed |> List.first() @@ -37,9 +36,9 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert blog.link_addr == link_addr end - @tag :wip - test "can create blog with no-exsit rss record", ~m(community user blog_attrs)a do - {:ok, feed} = CMS.blog_rss_feed(@rss) + @tag :wip2 + test "can create blog with no-exsit rss record", ~m(community user)a do + {:ok, feed} = CMS.blog_rss_info(@rss) selected_feed = feed.history_feed |> List.first() title = selected_feed |> Map.get(:title) @@ -56,12 +55,13 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert blog.link_addr == link_addr end - @tag :wip - test "can create blog with blog_author", ~m(community user blog_attrs)a do - {:ok, feed} = CMS.blog_rss_feed(@rss) + @tag :wip2 + test "can create blog with blog_author", ~m(community user)a do + {:ok, feed} = CMS.blog_rss_info(@rss) author = %{ name: "mydearxym", + intro: "this is mydearxym", link: "https://coderplaents.com" } @@ -86,14 +86,15 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert blog.title == title assert blog.link_addr == link_addr assert blog.blog_author.name == author.name + assert blog.blog_author.intro == author.intro assert blog.blog_author.link == author.link end end describe "fetch rss & curd" do - @tag :wip + @tag :wip2 test "parse and create basic rss" do - {:ok, feed} = CMS.blog_rss_feed(@rss) + {:ok, feed} = CMS.blog_rss_info(@rss) feed = feed |> Map.merge(%{rss: @rss}) {:ok, rss_record} = CMS.create_blog_rss(feed) @@ -103,9 +104,9 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert not is_nil(cache) end - @tag :wip + @tag :wip2 test "create rss with author" do - {:ok, feed} = CMS.blog_rss_feed(@rss) + {:ok, feed} = CMS.blog_rss_info(@rss) author = %{ name: "mydearxym", @@ -121,9 +122,9 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert rss_record.author.name == "mydearxym" end - @tag :wip + @tag :wip2 test "update rss with author and exsit feed" do - {:ok, feed} = CMS.blog_rss_feed(@rss) + {:ok, feed} = CMS.blog_rss_info(@rss) {:ok, rss_record} = CMS.create_blog_rss(feed) author = %{ From c932b697eda281ccab410ad92f4aea8191feb2e3 Mon Sep 17 00:00:00 2001 From: mydearxym Date: Thu, 30 Sep 2021 12:11:20 +0800 Subject: [PATCH 6/6] chore: wip --- test/groupher_server/seeds/articles_seed_test.exs | 1 + test/groupher_server_web/query/cms/blog_rss_test.exs | 3 +-- test/helper/rss_test.exs | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/test/groupher_server/seeds/articles_seed_test.exs b/test/groupher_server/seeds/articles_seed_test.exs index 8b3d54662..e0a683722 100644 --- a/test/groupher_server/seeds/articles_seed_test.exs +++ b/test/groupher_server/seeds/articles_seed_test.exs @@ -11,6 +11,7 @@ defmodule GroupherServer.Test.Seeds.Articles do alias Helper.ORM describe "[posts seed]" do + @tag :wip test "can seed posts" do {:ok, community} = CMS.seed_community(:home) CMS.seed_articles(community, :post, 5) diff --git a/test/groupher_server_web/query/cms/blog_rss_test.exs b/test/groupher_server_web/query/cms/blog_rss_test.exs index 95c81f8be..963ee97fc 100644 --- a/test/groupher_server_web/query/cms/blog_rss_test.exs +++ b/test/groupher_server_web/query/cms/blog_rss_test.exs @@ -34,7 +34,7 @@ defmodule GroupherServer.Test.Query.CMS.BlogRSS do } } """ - # @tag :wip2 + # test "basic graphql query blog rss info", ~m(user_conn)a do variables = %{rss: @rss} results = user_conn |> query_result(@query, variables, "blogRssInfo") @@ -42,7 +42,6 @@ defmodule GroupherServer.Test.Query.CMS.BlogRSS do assert not is_nil(results["title"]) end - @tag :wip test "invalid rss will get error", ~m(user_conn)a do variables = %{rss: "invalid rss address"} # results = user_conn |> query_result(@query, variables, "blogRssInfo") diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs index 2aa9ddd4a..6d630311d 100644 --- a/test/helper/rss_test.exs +++ b/test/helper/rss_test.exs @@ -16,7 +16,6 @@ defmodule GroupherServer.Test.Helper.RSSTest do end describe "blog curd" do - @tag :wip2 test "can create blog", ~m(community user)a do {:ok, feed} = CMS.blog_rss_info(@rss) {:ok, _rss_record} = CMS.create_blog_rss(feed) @@ -36,7 +35,6 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert blog.link_addr == link_addr end - @tag :wip2 test "can create blog with no-exsit rss record", ~m(community user)a do {:ok, feed} = CMS.blog_rss_info(@rss) @@ -55,7 +53,6 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert blog.link_addr == link_addr end - @tag :wip2 test "can create blog with blog_author", ~m(community user)a do {:ok, feed} = CMS.blog_rss_info(@rss) @@ -92,7 +89,6 @@ defmodule GroupherServer.Test.Helper.RSSTest do end describe "fetch rss & curd" do - @tag :wip2 test "parse and create basic rss" do {:ok, feed} = CMS.blog_rss_info(@rss) feed = feed |> Map.merge(%{rss: @rss}) @@ -104,7 +100,6 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert not is_nil(cache) end - @tag :wip2 test "create rss with author" do {:ok, feed} = CMS.blog_rss_info(@rss) @@ -122,7 +117,6 @@ defmodule GroupherServer.Test.Helper.RSSTest do assert rss_record.author.name == "mydearxym" end - @tag :wip2 test "update rss with author and exsit feed" do {:ok, feed} = CMS.blog_rss_info(@rss) {:ok, rss_record} = CMS.create_blog_rss(feed)