diff --git a/config/config.exs b/config/config.exs index 10b2bfb75..266ff0f54 100644 --- a/config/config.exs +++ b/config/config.exs @@ -134,10 +134,24 @@ config :groupher_server, GroupherServer.Mailer, adapter: Bamboo.MailgunAdapter, domain: "mailer.coderplanets.com" -# handle background jobs -config :rihanna, - jobs_table_name: "background_jobs", - producer_postgres_connection: {Ecto, GroupherServer.Repo} +config :groupher_server, :cache, + pool: %{ + common: %{ + name: :common, + size: 5000, + minutes: 10 + }, + user_login: %{ + name: :user_login, + size: 10_000, + minutes: 10_080 + }, + blog_rss: %{ + name: :blog_rss, + size: 1000, + minutes: 15 + } + } # cron-like job scheduler config :groupher_server, Helper.Scheduler, @@ -147,6 +161,11 @@ config :groupher_server, Helper.Scheduler, {"@daily", {Helper.Scheduler, :archive_artiments, []}} ] +# handle background jobs +config :rihanna, + jobs_table_name: "background_jobs", + producer_postgres_connection: {Ecto, GroupherServer.Repo} + import_config "#{Mix.env()}.exs" if File.exists?("config/#{Mix.env()}.secret.exs") do diff --git a/lib/groupher_server/application.ex b/lib/groupher_server/application.ex index ca13da300..cc6335dd1 100644 --- a/lib/groupher_server/application.ex +++ b/lib/groupher_server/application.ex @@ -1,30 +1,31 @@ defmodule GroupherServer.Application do @moduledoc false use Application + import Helper.Utils, only: [get_config: 2] + + alias Helper.Cache + + @cache_pool get_config(:cache, :pool) # See https://hexdocs.pm/elixir/Application.html # for more information on OTP Applications @spec start(any, any) :: {:error, any} | {:ok, pid} def start(_type, _args) do import Supervisor.Spec - alias Helper.Cache # Define workers and child supervisors to be supervised - children = [ - # Start the PubSub system - {Phoenix.PubSub, name: MyApp.PubSub}, - # Start the Ecto repository - supervisor(GroupherServer.Repo, []), - # Start the endpoint when the application starts - supervisor(GroupherServerWeb.Endpoint, []), - # Start your own worker by calling: GroupherServer.Worker.start_link(arg1, arg2, arg3) - # worker(GroupherServer.Worker, [arg1, arg2, arg3]), - worker(Cachex, [:common, Cache.config(:common)], id: :common), - worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login), - # - worker(Helper.Scheduler, []), - {Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]} - ] + children = + [ + # Start the PubSub system + {Phoenix.PubSub, name: MyApp.PubSub}, + # Start the Ecto repository + supervisor(GroupherServer.Repo, []), + # Start the endpoint when the application starts + supervisor(GroupherServerWeb.Endpoint, []), + # Start your own worker by calling: GroupherServer.Worker.start_link(arg1, arg2, arg3) + worker(Helper.Scheduler, []), + {Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]} + ] ++ cache_workers() # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options @@ -38,4 +39,19 @@ defmodule GroupherServer.Application do GroupherServerWeb.Endpoint.config_change(changed, removed) :ok end + + defp cache_workers() do + import Supervisor.Spec + + # worker(GroupherServer.Worker, [arg1, arg2, arg3]), + # worker(Cachex, [:common, Cache.config(:common)], id: :common), + # worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login), + # worker(Cachex, [:blog_rss, Cache.config(:blog_rss)], id: :blog_rss), + @cache_pool + |> Map.keys() + |> Enum.reduce([], fn key, acc -> + name = @cache_pool[key].name + acc ++ [worker(Cachex, [name, Cache.config(key)], id: name)] + end) + end end diff --git a/lib/groupher_server/cms/cms.ex b/lib/groupher_server/cms/cms.ex index fa5e43b99..be69a21b5 100644 --- a/lib/groupher_server/cms/cms.ex +++ b/lib/groupher_server/cms/cms.ex @@ -10,6 +10,7 @@ defmodule GroupherServer.CMS do alias Delegate.{ AbuseReport, ArticleCURD, + BlogCURD, ArticleCommunity, ArticleEmotion, CitedArtiment, @@ -102,6 +103,11 @@ defmodule GroupherServer.CMS do defdelegate archive_articles(thread), to: ArticleCURD + defdelegate create_blog(community, attrs, user), to: BlogCURD + defdelegate create_blog_rss(attrs), to: BlogCURD + defdelegate update_blog_rss(attrs), to: BlogCURD + defdelegate blog_rss_info(rss), to: BlogCURD + defdelegate paged_citing_contents(type, id, filter), to: CitedArtiment defdelegate upvote_article(thread, article_id, user), to: ArticleUpvote diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex new file mode 100644 index 000000000..beeea75a0 --- /dev/null +++ b/lib/groupher_server/cms/delegates/blog_curd.ex @@ -0,0 +1,137 @@ +defmodule GroupherServer.CMS.Delegate.BlogCURD do + @moduledoc """ + CURD operation on post/job ... + """ + import Ecto.Query, warn: false + import Helper.Utils, only: [strip_struct: 1, done: 1] + import Helper.ErrorCode + + import GroupherServer.CMS.Delegate.ArticleCURD, only: [create_article: 4] + # import Helper.Utils, only: [done: 1] + + # import Helper.ErrorCode + # import ShortMaps + + # alias Helper.{ORM} + alias GroupherServer.{Accounts, CMS, Repo} + alias CMS.Model.{BlogRSS, Community} + alias Accounts.Model.User + + alias Helper.{ORM, Cache, RSS} + + @cache_pool :blog_rss + + # alias Ecto.Multi + def blog_rss_info(rss) when is_binary(rss) do + with {:ok, feed} <- ORM.find_by(BlogRSS, %{rss: rss}) do + {:ok, feed} + else + _ -> fetch_fresh_rssinfo_and_cache(rss) + end + end + + # attrs 包含 rss, blog_title + # def create_article(%Community{id: cid}, thread, attrs, %User{id: uid}) do + def create_blog(%Community{} = community, attrs, %User{} = user) do + # 1. 先判断 rss 是否存在 + ## 1.1 如果存在，从 cache 中获取 + ## 1.2 如不存在，则创建一条 RSS + with {:ok, feed} <- blog_rss_info(attrs.rss) do + do_create_blog(community, attrs, user, feed) + + # IO.inspect(feed, label: "create blog") + # 通过 feed 有没有 id 来 insert / update + # 通过 blog_title, 组合 attrs 传给 create_article + end + + # 2. 创建 blog + ## 2.1 blog +字段 rss, author + ## 2.2 title, digest, xxx + + # 前台获取作者信息的时候从 rss 表读取 + end + + # rss 记录存在, 直接创建 blog + defp do_create_blog(%Community{} = community, attrs, %User{} = user, %{id: _} = feed) do + blog_author = if is_nil(feed.author), do: nil, else: Map.from_struct(feed.author) + selected_feed = Enum.find(feed.history_feed, &(&1.title == attrs.title)) + + # TODO: feed_digest, feed_content + attrs = + attrs + |> Map.merge(%{ + link_addr: selected_feed.link_addr, + published: selected_feed.published, + blog_author: blog_author + }) + |> Enum.reject(fn {_, v} -> is_nil(v) end) + |> Map.new() + + create_article(community, :blog, attrs, user) + end + + # rss 记录不存在, 先创建 rss, 再创建 blog + defp do_create_blog(%Community{} = community, attrs, %User{} = user, feed) do + with {:ok, feed} <- CMS.blog_rss_info(attrs.rss), + {:ok, feed} <- create_blog_rss(feed) do + do_create_blog(community, attrs, user, feed) + end + end + + def create_blog_rss(attrs) do + history_feed = Map.get(attrs, :history_feed) + attrs = attrs |> Map.drop([:history_feed]) + + %BlogRSS{} + |> Ecto.Changeset.change(attrs) + |> Ecto.Changeset.put_embed(:history_feed, history_feed) + |> Repo.insert() + end + + def update_blog_rss(%{rss: rss} = attrs) do + with {:ok, blog_rss} <- ORM.find_by(BlogRSS, rss: rss) do + history_feed = + Map.get(attrs, :history_feed, Enum.map(blog_rss.history_feed, &strip_struct(&1))) + + attrs = attrs |> Map.drop([:history_feed]) + + %BlogRSS{} + |> Ecto.Changeset.change(attrs) + |> Ecto.Changeset.put_embed(:history_feed, history_feed) + |> Repo.insert() + end + end + + # create done + # defp result({:ok, %{set_active_at_timestamp: result}}) do + # {:ok, result} + # end + + # defp result({:ok, %{update_article_meta: result}}), do: {:ok, result} + + # defp result({:error, :create_article, _result, _steps}) do + # {:error, [message: "create article", code: ecode(:create_fails)]} + # end + + # defp result({:error, _, result, _steps}), do: {:error, result} + + @doc """ + get and cache feed by rss address as key + """ + def fetch_fresh_rssinfo_and_cache(rss) do + case Cache.get(@cache_pool, rss) do + {:ok, rssinfo} -> {:ok, rssinfo} + {:error, _} -> get_rssinfo_and_cache(rss) + end + end + + defp get_rssinfo_and_cache(rss) do + # {:ok, feed} = RSS.get(rss) + with {:ok, rssinfo} <- RSS.get(rss) do + Cache.put(@cache_pool, rss, rssinfo) + {:ok, rssinfo} + else + {:error, _} -> {:error, [message: "blog rss is invalid", code: ecode(:invalid_blog_rss)]} + end + end +end diff --git a/lib/groupher_server/cms/models/blog.ex b/lib/groupher_server/cms/models/blog.ex index 032ef3930..225333f24 100644 --- a/lib/groupher_server/cms/models/blog.ex +++ b/lib/groupher_server/cms/models/blog.ex @@ -15,13 +15,18 @@ defmodule GroupherServer.CMS.Model.Blog do @required_fields ~w(title digest)a @article_cast_fields general_article_cast_fields() - @optional_fields ~w(digest)a ++ @article_cast_fields + @optional_fields ~w(digest feed_digest feed_content published)a ++ @article_cast_fields @type t :: %Blog{} schema "cms_blogs" do # for frontend constant field(:copy_right, :string, default: "", virtual: true) + field(:feed_digest, :string) + field(:feed_content, :string) + field(:published, :string) + embeds_one(:blog_author, Embeds.BlogAuthor, on_replace: :update) + article_tags_field(:blog) article_communities_field(:blog) general_article_fields(:blog) @@ -33,6 +38,7 @@ defmodule GroupherServer.CMS.Model.Blog do |> cast(attrs, @optional_fields ++ @required_fields) |> validate_required(@required_fields) |> cast_embed(:meta, required: false, with: &Embeds.ArticleMeta.changeset/2) + |> cast_embed(:blog_author, required: false, with: &Embeds.BlogAuthor.changeset/2) |> generl_changeset end @@ -40,6 +46,7 @@ defmodule GroupherServer.CMS.Model.Blog do def update_changeset(%Blog{} = blog, attrs) do blog |> cast(attrs, @optional_fields ++ @required_fields) + |> cast_embed(:blog_author, required: false, with: &Embeds.BlogAuthor.changeset/2) |> generl_changeset end diff --git a/lib/groupher_server/cms/models/blog_rss.ex b/lib/groupher_server/cms/models/blog_rss.ex new file mode 100644 index 000000000..fdf65288d --- /dev/null +++ b/lib/groupher_server/cms/models/blog_rss.ex @@ -0,0 +1,60 @@ +defmodule GroupherServer.CMS.Model.BlogRSS do + @moduledoc false + alias __MODULE__ + + use Ecto.Schema + use Accessible + + import Ecto.Changeset + # import GroupherServer.CMS.Helper.Macros + + alias GroupherServer.CMS + alias CMS.Model.Embeds + + @timestamps_opts [type: :utc_datetime_usec] + + @required_fields ~w(link rss)a + @optional_fields ~w(subtitle author updated)a + + @type t :: %BlogRSS{} + schema "cms_blog_rss" do + field(:rss, :string) + field(:title, :string) + field(:subtitle, :string) + field(:link, :string) + field(:updated, :string) + embeds_many(:history_feed, Embeds.BlogHistoryFeed, on_replace: :delete) + embeds_one(:author, Embeds.BlogAuthor, on_replace: :update) + end + + @doc false + def changeset(%BlogRSS{} = blog_rss, attrs) do + blog_rss + |> cast(attrs, @optional_fields ++ @required_fields) + |> validate_required(@required_fields) + |> cast_embed(:history_feed, required: true, with: &Embeds.BlogHistoryFeed.changeset/2) + |> cast_embed(:author, required: false, with: &Embeds.BlogAuthor.changeset/2) + end + + @doc false + def update_changeset(%BlogRSS{} = blog_rss, attrs) do + blog_rss + |> cast(attrs, @optional_fields ++ @required_fields) + |> cast_embed(:history_feed, required: false, with: &Embeds.BlogHistoryFeed.changeset/2) + |> cast_embed(:author, required: false, with: &Embeds.BlogAuthor.changeset/2) + end + + # @doc false + # def update_changeset(%BlogRSS{} = blog_rss, attrs) do + # blog_rss + # |> cast(attrs, @optional_fields ++ @required_fields) + # |> generl_changeset + # end + + # defp generl_changeset(changeset) do + # changeset + # |> validate_length(:title, min: 3, max: 100) + # |> cast_embed(:emotions, with: &Embeds.ArticleEmotion.changeset/2) + # |> validate_length(:link_addr, min: 5, max: 400) + # end +end diff --git a/lib/groupher_server/cms/models/embeds/blog_author.ex b/lib/groupher_server/cms/models/embeds/blog_author.ex new file mode 100644 index 000000000..6bc1ed296 --- /dev/null +++ b/lib/groupher_server/cms/models/embeds/blog_author.ex @@ -0,0 +1,26 @@ +defmodule GroupherServer.CMS.Model.Embeds.BlogAuthor do + @moduledoc """ + general community meta + """ + use Ecto.Schema + use Accessible + + import Ecto.Changeset + + @required_fields ~w(name)a + @optional_fields ~w(link intro github twitter)a + + embedded_schema do + field(:name, :string) + field(:link, :string) + field(:intro, :string) + field(:github, :string) + field(:twitter, :string) + end + + def changeset(struct, attrs) do + struct + |> cast(attrs, @optional_fields ++ @required_fields) + |> validate_required(@required_fields) + end +end diff --git a/lib/groupher_server/cms/models/embeds/blog_history_feed.ex b/lib/groupher_server/cms/models/embeds/blog_history_feed.ex new file mode 100644 index 000000000..801ba71ec --- /dev/null +++ b/lib/groupher_server/cms/models/embeds/blog_history_feed.ex @@ -0,0 +1,25 @@ +defmodule GroupherServer.CMS.Model.Embeds.BlogHistoryFeed do + @moduledoc """ + general community meta + """ + use Ecto.Schema + use Accessible + + import Ecto.Changeset + + @optional_fields ~w(title digest link_addr content published_at)a + + embedded_schema do + field(:title, :string) + field(:digest, :string) + field(:link_addr, :string) + field(:content, :string) + field(:published, :string) + field(:updated, :string) + end + + def changeset(struct, params) do + struct + |> cast(params, @optional_fields) + end +end diff --git a/lib/groupher_server_web/resolvers/cms_resolver.ex b/lib/groupher_server_web/resolvers/cms_resolver.ex index b29c72c39..58be1c35c 100644 --- a/lib/groupher_server_web/resolvers/cms_resolver.ex +++ b/lib/groupher_server_web/resolvers/cms_resolver.ex @@ -62,6 +62,11 @@ defmodule GroupherServerWeb.Resolvers.CMS do CMS.paged_reports(filter) end + # TODO: login only + def blog_rss_info(_root, ~m(rss)a, _) do + CMS.blog_rss_info(rss) + end + def wiki(_root, ~m(community)a, _info), do: CMS.get_wiki(%Community{raw: community}) def cheatsheet(_root, ~m(community)a, _info), do: CMS.get_cheatsheet(%Community{raw: community}) diff --git a/lib/groupher_server_web/schema/cms/cms_queries.ex b/lib/groupher_server_web/schema/cms/cms_queries.ex index 6ed0a3284..33b1b6baf 100644 --- a/lib/groupher_server_web/schema/cms/cms_queries.ex +++ b/lib/groupher_server_web/schema/cms/cms_queries.ex @@ -140,6 +140,13 @@ defmodule GroupherServerWeb.Schema.CMS.Queries do resolve(&R.CMS.search_communities/3) end + @desc "get rss info based on blog rss address" + field :blog_rss_info, :blog_rss do + arg(:rss, non_null(:string)) + + resolve(&R.CMS.blog_rss_info/3) + end + article_search_queries() article_reacted_users_query(:upvot, &R.CMS.upvoted_users/3) diff --git a/lib/groupher_server_web/schema/cms/cms_types.ex b/lib/groupher_server_web/schema/cms/cms_types.ex index adb0ade7f..202bc3c33 100644 --- a/lib/groupher_server_web/schema/cms/cms_types.ex +++ b/lib/groupher_server_web/schema/cms/cms_types.ex @@ -347,6 +347,32 @@ defmodule GroupherServerWeb.Schema.CMS.Types do timestamp_fields() end + object :blog_feed do + field(:title, :string) + field(:digest, :string) + field(:link_addr, :string) + field(:content, :string) + field(:published, :string) + field(:updated, :string) + end + + object :blog_author do + field(:name, :string) + field(:intro, :string) + field(:github, :string) + field(:twitter, :string) + end + + object :blog_rss do + field(:rss, :string) + field(:title, :string) + field(:subtitle, :string) + field(:link, :string) + field(:updated, :string) + field(:author, :blog_author) + field(:history_feed, list_of(:blog_feed)) + end + paged_article_objects() object :paged_reports do diff --git a/lib/helper/RSS.ex b/lib/helper/RSS.ex new file mode 100644 index 000000000..c0f45dc99 --- /dev/null +++ b/lib/helper/RSS.ex @@ -0,0 +1,115 @@ +defmodule Helper.RSS do + @moduledoc """ + RSS get and parser + """ + import Helper.Utils, only: [done: 1] + + def get(rss) do + with {:ok, %{body: body}} <- HTTPoison.get(rss), + {:ok, blog_rss} <- rss_parser(body) do + blog_rss |> Map.merge(%{rss: rss}) |> done + else + error -> + IO.inspect(error, label: "error") + {:error, :invalid_rss_address} + end + end + + defp rss_parser(body) do + with {:ok, feed} <- Fiet.Atom.parse(body) do + # IO.inspect(feed, label: "atom feed") + format(:atom, feed) + else + {:error, %Fiet.Atom.ParsingError{reason: {:not_atom, "rss"}}} -> + rss_parser(body, :rss2) + end + end + + defp rss_parser(body, :rss2) do + with {:ok, feed} <- Fiet.RSS2.parse(body) do + # IO.inspect(feed, label: "rss2 feed") + format(:rss2, feed) + end + end + + defp format(:atom, %Fiet.Atom.Feed{entries: entries} = feed) do + items = + Enum.reduce(entries, [], fn item, acc -> + acc ++ [format(:item, item)] + end) + + {:ok, + %{ + title: parse(:text, feed.title), + subtitle: parse(:text, feed.subtitle), + link: parse(:link, feed), + updated: feed.updated, + history_feed: items + }} + end + + defp format(:rss2, %Fiet.RSS2.Channel{items: items} = feed) do + items = + Enum.reduce(items, [], fn item, acc -> + acc ++ [format(:item, item)] + end) + + {:ok, + %{ + title: feed.title, + subtitle: feed.description, + link: feed.link, + updated: feed.last_build_date, + history_feed: items + }} + end + + defp format(:item, %Fiet.Atom.Entry{} = item) do + %{ + title: parse(:text, item.title), + digest: parse(:digest, item), + link_addr: parse(:link, item), + # + published: parse(:published, item), + updated: item.updated + } + end + + defp format(:item, %Fiet.RSS2.Item{} = item) do + %{ + title: item.title, + digest: item.description, + link_addr: item.link, + # + published: item.pub_date, + updated: item.pub_date + } + end + + defp parse(:digest, %Fiet.Atom.Entry{summary: nil}), do: "use content TODO" + defp parse(:digest, %Fiet.Atom.Entry{summary: summary}), do: parse(:text, summary) + defp parse(:digest, _), do: "use content TODO" + + defp parse(:text, {:text, text}), do: text + defp parse(:text, _), do: "" + + defp parse(:link, %Fiet.Atom.Entry{links: links}), do: do_parse_link(links) + defp parse(:link, %Fiet.Atom.Feed{links: links}), do: do_parse_link(links) + + defp parse(:published, %Fiet.Atom.Entry{published: nil, updated: updated}) do + updated + end + + defp parse(:published, %Fiet.Atom.Entry{published: published}) do + published + end + + defp do_parse_link([]), do: "" + + defp do_parse_link(links) do + case Enum.find(links, &(&1.type === "text/html")) do + nil -> links |> List.first() |> Map.get(:href) + link -> link.href + end + end +end diff --git a/lib/helper/cache.ex b/lib/helper/cache.ex index 7f5f50857..ee5374694 100644 --- a/lib/helper/cache.ex +++ b/lib/helper/cache.ex @@ -3,26 +3,45 @@ defmodule Helper.Cache do memory cache using cachex https://github.com/whitfin/cachex """ import Cachex.Spec + import Helper.Utils, only: [get_config: 2] - def config(:common) do - [ - limit: limit(size: 5000, policy: Cachex.Policy.LRW, reclaim: 0.1), - expiration: expiration(default: :timer.minutes(10)) - ] - end + @cache_pool get_config(:cache, :pool) - @doc """ - cache config for user.login -> user.id, used in accounts resolver - user.id is a linearly increasing integer, kind sensitive, so use user.login instead - """ - def config(:user_login) do + def config(pool_name) do [ - limit: limit(size: 10_000, policy: Cachex.Policy.LRW, reclaim: 0.1), - # expired in one week, it's fine, since user's login and id will never change - expiration: expiration(default: :timer.minutes(10_080)) + limit: limit(size: @cache_pool[pool_name].size, policy: Cachex.Policy.LRW, reclaim: 0.1), + expiration: expiration(default: :timer.minutes(@cache_pool[pool_name].minutes)) ] end + # # size, minites + # def config(:common) do + # [ + # limit: limit(size: 5000, policy: Cachex.Policy.LRW, reclaim: 0.1), + # expiration: expiration(default: :timer.minutes(10)) + # ] + # end + + # @doc """ + # cache config for user.login -> user.id, used in accounts resolver + # user.id is a linearly increasing integer, kind sensitive, so use user.login instead + # """ + # def config(:user_login) do + # [ + # limit: limit(size: 10_000, policy: Cachex.Policy.LRW, reclaim: 0.1), + # # expired in one week, it's fine, since user's login and id will never change + # expiration: expiration(default: :timer.minutes(10_080)) + # ] + # end + + # def config(:blog_rss) do + # [ + # limit: limit(size: 1000, policy: Cachex.Policy.LRW, reclaim: 0.1), + # # expired in one week, it's fine, since user's login and id will never change + # expiration: expiration(default: :timer.minutes(10)) + # ] + # end + @doc """ ## Example iex> Helper.Cache.get(:common, :a) diff --git a/lib/helper/error_code.ex b/lib/helper/error_code.ex index 6be1a230d..e947c7c16 100644 --- a/lib/helper/error_code.ex +++ b/lib/helper/error_code.ex @@ -53,6 +53,7 @@ defmodule Helper.ErrorCode do def ecode(:require_questioner), do: @article_base + 9 def ecode(:cite_artilce), do: @article_base + 10 def ecode(:archived), do: @article_base + 11 + def ecode(:invalid_blog_rss), do: @article_base + 12 # def ecode(:already_solved), do: @article_base + 10 def ecode, do: @default_base diff --git a/mix.exs b/mix.exs index 690c7708b..ecbb0136b 100644 --- a/mix.exs +++ b/mix.exs @@ -110,7 +110,10 @@ defmodule GroupherServer.Mixfile do # https://github.com/cataska/pangu.ex {:pangu, "~> 0.1.0"}, {:accessible, "~> 0.3.0"}, - {:floki, "~> 0.30.1"} + {:floki, "~> 0.30.1"}, + {:httpoison, "~> 1.8"}, + # rss feed parser + {:fiet, "~> 0.3"} ] end diff --git a/mix.lock b/mix.lock index 8ab81947b..da923df79 100644 --- a/mix.lock +++ b/mix.lock @@ -37,6 +37,7 @@ "ex_zstd": {:hex, :ex_zstd, "0.1.0", "4b1b5ebd7c0417e69308db8cdd478b9adb3e2d1a03b6e7366cf0a9aadeae11af", [:make, :mix], [{:ex_doc, ">= 0.0.0", [hex: :ex_doc, repo: "hexpm", optional: false]}], "hexpm", "2c9542a5c088e0eab14aa9b10d18bc084a6060ecf09025bbfc5b08684568bc67"}, "excoveralls": {:hex, :excoveralls, "0.14.1", "14140e4ef343f2af2de33d35268c77bc7983d7824cb945e6c2af54235bc2e61f", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "4a588f9f8cf9dc140cc1f3d0ea4d849b2f76d5d8bee66b73c304bb3d3689c8b0"}, "faker": {:hex, :faker, "0.16.0", "1e2cf3e8d60d44a30741fb98118fcac18b2020379c7e00d18f1a005841b2f647", [:mix], [], "hexpm", "fbcb9bf1299dff3c9dd7e50f41802bbc472ffbb84e7656394c8aa913ec315141"}, + "fiet": {:hex, :fiet, "0.3.0", "fdfc03119250e7f4b2eac88a03325d43af80b08e55aa80352d0d138a972815f9", [:mix], [{:saxy, "~> 1.2", [hex: :saxy, repo: "hexpm", optional: false]}], "hexpm", "b5cff4338b0b216c04b489dd99c839524469d7d21cd6dc9717768497276810eb"}, "file_system": {:hex, :file_system, "0.2.8", "f632bd287927a1eed2b718f22af727c5aeaccc9a98d8c2bd7bff709e851dc986", [:mix], [], "hexpm", "97a3b6f8d63ef53bd0113070102db2ce05352ecf0d25390eb8d747c2bde98bca"}, "floki": {:hex, :floki, "0.30.1", "75d35526d3a1459920b6e87fdbc2e0b8a3670f965dd0903708d2b267e0904c55", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e9c03524447d1c4cbfccd672d739b8c18453eee377846b119d4fd71b1a176bb8"}, "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"}, @@ -47,7 +48,7 @@ "hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.4.1", "e8a67da405fe9f0d1be121a40a60f70811192033a5b8d00a95dddd807f5e053e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm", "68d92656f47cd73598c45ad2394561f025c8c65d146001b955fd7b517858962a"}, - "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, + "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inch_ex": {:hex, :inch_ex, "2.0.0", "24268a9284a1751f2ceda569cd978e1fa394c977c45c331bb52a405de544f4de", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "96d0ec5ecac8cf63142d02f16b7ab7152cf0f0f1a185a80161b758383c9399a8"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, @@ -86,6 +87,7 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, "recase": {:hex, :recase, "0.7.0", "3f2f719f0886c7a3b7fe469058ec539cb7bbe0023604ae3bce920e186305e5ae", [:mix], [], "hexpm", "36f5756a9f552f4a94b54a695870e32f4e72d5fad9c25e61bc4a3151c08a4e0c"}, "rihanna": {:hex, :rihanna, "1.3.5", "5f5e6c5b1e514978a29a6791f338f4bb963401959fc212bd18d4a2c92d79a7a4", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.13.3", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "fa1918c2ab63c8ada9a23ad6fe03cd181378739f0ff10741b45d0bcb50003c74"}, + "saxy": {:hex, :saxy, "1.4.0", "c7203ad20001f72eaaad07d08f82be063fa94a40924e6bb39d93d55f979abcba", [:mix], [], "hexpm", "3fe790354d3f2234ad0b5be2d99822a23fa2d4e8ccd6657c672901dac172e9a9"}, "scrivener": {:hex, :scrivener, "2.5.0", "e1f78c62b6806d91cc9c4778deef1ea4e80aa9fadfce2c16831afe0468cc8a2c", [:mix], [], "hexpm", "c3e484da6bb7084b5a24c7e38a8ca09310d5fbf5241db05f625fb8af557ef667"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, "sentry": {:hex, :sentry, "7.1.0", "546729ea0be4a3f593b456fe77a2cf5537e390fbe87c191424557dae8c2bd760", [:mix], [{:hackney, "~> 1.8 or 1.6.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "6e439c2b93a3d1e049eeaa7910d9ef4ff255b107b2192c25fe50e5f2cd57d2d3"}, diff --git a/priv/repo/migrations/20210924023609_create_blog_rss.exs b/priv/repo/migrations/20210924023609_create_blog_rss.exs new file mode 100644 index 000000000..c112c445a --- /dev/null +++ b/priv/repo/migrations/20210924023609_create_blog_rss.exs @@ -0,0 +1,17 @@ +defmodule GroupherServer.Repo.Migrations.CreateBlogRss do + use Ecto.Migration + + def change do + create table(:cms_blog_rss) do + add(:rss, :string) + add(:link, :string) + add(:title, :string) + add(:subtitle, :string) + add(:updated, :string) + add(:history_feed, :map) + add(:author, :map) + end + + create(unique_index(:cms_blog_rss, [:rss, :link])) + end +end diff --git a/priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs b/priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs new file mode 100644 index 000000000..a049c957a --- /dev/null +++ b/priv/repo/migrations/20210926015205_add_feed_fields_to_blog.exs @@ -0,0 +1,12 @@ +defmodule GroupherServer.Repo.Migrations.AddFeedFieldsToBlog do + use Ecto.Migration + + def change do + alter table(:cms_blogs) do + add(:feed_digest, :string) + add(:feed_content, :text) + add(:published, :string) + add(:blog_author, :map) + end + end +end diff --git a/test/groupher_server/cms/article_tags/post_tag_test.exs b/test/groupher_server/cms/article_tags/post_tag_test.exs index a69b6186b..fdf45468d 100644 --- a/test/groupher_server/cms/article_tags/post_tag_test.exs +++ b/test/groupher_server/cms/article_tags/post_tag_test.exs @@ -139,7 +139,7 @@ defmodule GroupherServer.Test.CMS.ArticleTag.PostTag do assert not exist_in?(article_tag2, post.article_tags) end - test "can not set dup tag ", ~m(community post article_tag_attrs article_tag_attrs2 user)a do + test "can not set dup tag ", ~m(community post article_tag_attrs user)a do {:ok, article_tag} = CMS.create_article_tag(community, :post, article_tag_attrs, user) {:ok, post} = CMS.set_article_tag(:post, post.id, article_tag.id) {:ok, post} = CMS.set_article_tag(:post, post.id, article_tag.id) diff --git a/test/groupher_server/seeds/articles_seed_test.exs b/test/groupher_server/seeds/articles_seed_test.exs index 8b3d54662..e0a683722 100644 --- a/test/groupher_server/seeds/articles_seed_test.exs +++ b/test/groupher_server/seeds/articles_seed_test.exs @@ -11,6 +11,7 @@ defmodule GroupherServer.Test.Seeds.Articles do alias Helper.ORM describe "[posts seed]" do + @tag :wip test "can seed posts" do {:ok, community} = CMS.seed_community(:home) CMS.seed_articles(community, :post, 5) diff --git a/test/groupher_server/seeds/clean_up_test.exs b/test/groupher_server/seeds/clean_up_test.exs index 527cafe40..af6d9c2d4 100644 --- a/test/groupher_server/seeds/clean_up_test.exs +++ b/test/groupher_server/seeds/clean_up_test.exs @@ -22,7 +22,7 @@ defmodule GroupherServer.Test.Seeds.CleanUp do describe "[community clean up]" do test "can clean up a community", ~m(user post_attrs)a do {:ok, community} = CMS.seed_community(:home) - {:ok, post} = CMS.create_article(community, :post, post_attrs, user) + {:ok, _post} = CMS.create_article(community, :post, post_attrs, user) {:ok, found} = ORM.find_all(ArticleTag, %{page: 1, size: 20}) assert found.total_count !== 0 diff --git a/test/groupher_server_web/query/cms/blog_rss_test.exs b/test/groupher_server_web/query/cms/blog_rss_test.exs new file mode 100644 index 000000000..963ee97fc --- /dev/null +++ b/test/groupher_server_web/query/cms/blog_rss_test.exs @@ -0,0 +1,51 @@ +defmodule GroupherServer.Test.Query.CMS.BlogRSS do + use GroupherServer.TestTools + + @rss mock_rss_addr() + + setup do + guest_conn = simu_conn(:guest) + user_conn = simu_conn(:user) + + {:ok, ~m(user_conn guest_conn)a} + end + + @query """ + query($rss: String!) { + blogRssInfo(rss: $rss) { + title + subtitle + link + updated + author { + name + intro + github + twitter + } + historyFeed { + title + digest + linkAddr + content + published + updated + } + } + } + """ + # + test "basic graphql query blog rss info", ~m(user_conn)a do + variables = %{rss: @rss} + results = user_conn |> query_result(@query, variables, "blogRssInfo") + + assert not is_nil(results["title"]) + end + + test "invalid rss will get error", ~m(user_conn)a do + variables = %{rss: "invalid rss address"} + # results = user_conn |> query_result(@query, variables, "blogRssInfo") + assert user_conn |> query_get_error?(@query, variables, ecode(:invalid_blog_rss)) + # IO.inspect(results, label: "iiii") + end +end diff --git a/test/groupher_server_web/query/cms/comments/guide_comment_test.exs b/test/groupher_server_web/query/cms/comments/guide_comment_test.exs index 5974ccf64..c21568777 100644 --- a/test/groupher_server_web/query/cms/comments/guide_comment_test.exs +++ b/test/groupher_server_web/query/cms/comments/guide_comment_test.exs @@ -30,7 +30,6 @@ defmodule GroupherServer.Test.Query.Comments.GuideComment do } } """ - @tag :wip test "guest user can get comment participants after comment created", ~m(guest_conn guide user user2)a do total_count = 5 diff --git a/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs b/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs index 439482bb3..7fbe74a75 100644 --- a/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs +++ b/test/groupher_server_web/query/cms/comments/meetup_comment_test.exs @@ -193,7 +193,6 @@ defmodule GroupherServer.Test.Query.Comments.MeetupComment do assert random_comment["repliesCount"] == 2 end - @tag :wip test "comment should have reply_to content if need", ~m(guest_conn meetup user user2)a do total_count = 2 thread = :meetup diff --git a/test/helper/rss_test.exs b/test/helper/rss_test.exs new file mode 100644 index 000000000..6d630311d --- /dev/null +++ b/test/helper/rss_test.exs @@ -0,0 +1,134 @@ +defmodule GroupherServer.Test.Helper.RSSTest do + @moduledoc false + use GroupherServer.TestTools + + alias GroupherServer.CMS + alias Helper.{Cache} + + @cache_pool :blog_rss + @rss mock_rss_addr() + + setup do + {:ok, community} = db_insert(:community) + {:ok, user} = db_insert(:user) + + {:ok, ~m(community user)a} + end + + describe "blog curd" do + test "can create blog", ~m(community user)a do + {:ok, feed} = CMS.blog_rss_info(@rss) + {:ok, _rss_record} = CMS.create_blog_rss(feed) + + selected_feed = feed.history_feed |> List.first() + title = selected_feed |> Map.get(:title) + link_addr = selected_feed |> Map.get(:link_addr) + # blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + blog_attrs = %{ + rss: @rss, + title: title, + body: mock_rich_text("pleace use content field instead") + } + + {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + assert blog.title == title + assert blog.link_addr == link_addr + end + + test "can create blog with no-exsit rss record", ~m(community user)a do + {:ok, feed} = CMS.blog_rss_info(@rss) + + selected_feed = feed.history_feed |> List.first() + title = selected_feed |> Map.get(:title) + link_addr = selected_feed |> Map.get(:link_addr) + # blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + blog_attrs = %{ + rss: @rss, + title: title, + body: mock_rich_text("pleace use content field instead") + } + + {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + assert blog.title == title + assert blog.link_addr == link_addr + end + + test "can create blog with blog_author", ~m(community user)a do + {:ok, feed} = CMS.blog_rss_info(@rss) + + author = %{ + name: "mydearxym", + intro: "this is mydearxym", + link: "https://coderplaents.com" + } + + feed = + feed + |> Map.merge(%{rss: @rss}) + |> Map.merge(%{author: author}) + + {:ok, _rss_record} = CMS.create_blog_rss(feed) + + selected_feed = feed.history_feed |> List.first() + title = selected_feed |> Map.get(:title) + link_addr = selected_feed |> Map.get(:link_addr) + # blog_attrs = mock_attrs(:blog, %{community_id: community.id}) + blog_attrs = %{ + rss: @rss, + title: title, + body: mock_rich_text("pleace use content field instead") + } + + {:ok, blog} = CMS.create_blog(community, blog_attrs, user) + assert blog.title == title + assert blog.link_addr == link_addr + assert blog.blog_author.name == author.name + assert blog.blog_author.intro == author.intro + assert blog.blog_author.link == author.link + end + end + + describe "fetch rss & curd" do + test "parse and create basic rss" do + {:ok, feed} = CMS.blog_rss_info(@rss) + feed = feed |> Map.merge(%{rss: @rss}) + + {:ok, rss_record} = CMS.create_blog_rss(feed) + assert rss_record.history_feed |> length !== 0 + + {:ok, cache} = Cache.get(@cache_pool, @rss) + assert not is_nil(cache) + end + + test "create rss with author" do + {:ok, feed} = CMS.blog_rss_info(@rss) + + author = %{ + name: "mydearxym", + link: "https://coderplaents.com" + } + + feed = + feed + |> Map.merge(%{rss: @rss}) + |> Map.merge(%{author: author}) + + {:ok, rss_record} = CMS.create_blog_rss(feed) + assert rss_record.author.name == "mydearxym" + end + + test "update rss with author and exsit feed" do + {:ok, feed} = CMS.blog_rss_info(@rss) + {:ok, rss_record} = CMS.create_blog_rss(feed) + + author = %{ + name: "mydearxym", + link: "https://coderplaents.com" + } + + attrs = %{rss: rss_record.rss, author: author, history_feed: rss_record.history_feed} + {:ok, rss_record} = CMS.update_blog_rss(attrs) + assert rss_record.author.name == "mydearxym" + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index e4ead7c9a..c6ed4f426 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -539,6 +539,15 @@ defmodule GroupherServer.Support.Factory do @images |> Enum.slice(0, count) end + def mock_rss_addr() do + # "https://www.xiabingbao.com/atom.xml" # 不规范 + # "https://rsshub.app/blogs/wangyin" + "https://www.zhangxinxu.com/wordpress/feed/" + # "https://overreacted.io/rss.xml" + # "https://www.ruanyifeng.com/blog/atom.xml" + # "https://lutaonan.com/rss.xml" + end + def mock_mention_for(user, from_user) do {:ok, post} = db_insert(:post)