diff --git a/config/config.exs b/config/config.exs index 26ca1307a..618490c1b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -80,6 +80,13 @@ config :rihanna, jobs_table_name: "background_jobs", producer_postgres_connection: {Ecto, GroupherServer.Repo} +# cron-like job scheduler +config :groupher_server, Helper.Scheduler, + jobs: [ + # Every midnight + {"@daily", {Helper.Scheduler, :clear_all_cache, []}} + ] + import_config "#{Mix.env()}.exs" if File.exists?("config/#{Mix.env()}.secret.exs") do diff --git a/lib/groupher_server/application.ex b/lib/groupher_server/application.ex index d7cc249d3..092b64a92 100644 --- a/lib/groupher_server/application.ex +++ b/lib/groupher_server/application.ex @@ -32,6 +32,7 @@ defmodule GroupherServer.Application do ) ] ]), + worker(Helper.Scheduler, []), {Rihanna.Supervisor, [postgrex: GroupherServer.Repo.config()]} ] diff --git a/lib/groupher_server/statistics/delegates/contribute.ex b/lib/groupher_server/statistics/delegates/contribute.ex index e8fbbb51c..db15d51c0 100644 --- a/lib/groupher_server/statistics/delegates/contribute.ex +++ b/lib/groupher_server/statistics/delegates/contribute.ex @@ -85,7 +85,7 @@ defmodule GroupherServer.Statistics.Delegate.Contribute do %Community{id: id} |> do_get_contributes() |> to_counts_digest(days: @community_contribute_days) - |> done_and_cache(scope) + |> done_and_cache(scope, expire: 60_000) end defp update_contribute_record(%UserContribute{} = contribute) do diff --git a/lib/helper/cache.ex b/lib/helper/cache.ex index 9bc5269e1..ef4907cfe 100644 --- a/lib/helper/cache.ex +++ b/lib/helper/cache.ex @@ -27,6 +27,19 @@ defmodule Helper.Cache do Cachex.put(:site_cache, cache_key, cache_value) end + def put(cache_key, cache_value, expire: expire_time) do + Cachex.put(:site_cache, cache_key, cache_value) + Cachex.expire(:site_cache, cache_key, expire_time) + end + + @doc """ + clear all the cache + ## Example + iex> Helper.Cache.clear() + {:ok, 1} + """ + def clear_all(), do: Cachex.clear(:site_cache) + @doc """ cache scope of community contributes digest """ diff --git a/lib/helper/scheduler.ex b/lib/helper/scheduler.ex new file mode 100644 index 000000000..a8fdc0f16 --- /dev/null +++ b/lib/helper/scheduler.ex @@ -0,0 +1,16 @@ +defmodule Helper.Scheduler do + @moduledoc """ + cron-like job scheduler + """ + use Quantum.Scheduler, otp_app: :groupher_server + + alias Helper.Cache + + @doc """ + clear all the cache in Cachex + just in case the cache system broken + """ + def clear_all_cache do + Cache.clear_all() + end +end diff --git a/lib/helper/utils.ex b/lib/helper/utils.ex index b417d46fe..9755fa307 100644 --- a/lib/helper/utils.ex +++ b/lib/helper/utils.ex @@ -52,6 +52,13 @@ defmodule Helper.Utils do # def done({:error, error}), do: {:error, error} def done(result), do: {:ok, result} + def done_and_cache(result, scope, expire: expire_time) do + with {:ok, res} <- done(result) do + Cache.put(scope, res, expire: expire_time) + {:ok, res} + end + end + def done_and_cache(result, scope) do with {:ok, res} <- done(result) do Cache.put(scope, res) diff --git a/mix.exs b/mix.exs index 4cc9383cd..3bc63e3b5 100644 --- a/mix.exs +++ b/mix.exs @@ -93,9 +93,14 @@ defmodule GroupherServer.Mixfile do {:sentry, "~> 7.1"}, {:recase, "~> 0.4.0"}, {:nanoid, "~> 2.0.0"}, + # mailer {:bamboo, "1.2.0"}, + # mem cache {:cachex, "3.1.3"}, - {:rihanna, "1.3.5"} + # postgres-backed job queue + {:rihanna, "1.3.5"}, + # cron-like scheduler job + {:quantum, "~> 2.3"} ] end diff --git a/mix.lock b/mix.lock index e2eddc54a..1e0b27537 100644 --- a/mix.lock +++ b/mix.lock @@ -17,6 +17,7 @@ "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "1.1.0", "e0c07b2fd7e2109495f582430a1bc96b2c71b7d94c59dfad120529f65f19872f", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "dataloader": {:hex, :dataloader, "1.0.6", "fb724d6d3fb6acb87d27e3b32dea3a307936ad2d245faf9cf5221d1323d6a4ba", [:mix], [{:ecto, ">= 0.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "db_connection": {:hex, :db_connection, "2.1.0", "122e2f62c4906bf2e49554f1e64db5030c19229aa40935f33088e7d543aa79d0", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, @@ -31,6 +32,8 @@ "faker": {:hex, :faker, "0.10.0", "367c2ae47e7b4ac6410e1eaa880c07b5fe4194476f697d37ac1ce25f3058aae2", [:mix], [], "hexpm"}, "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"}, "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"}, + "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, "guardian": {:hex, :guardian, "1.2.1", "bdc8dd3dbf0fb7216cb6f91c11831faa1a64d39cdaed9a611e37f2413e584983", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, @@ -39,6 +42,7 @@ "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm"}, + "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, @@ -57,6 +61,7 @@ "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "pre_commit": {:hex, :pre_commit, "0.3.4", "e2850f80be8090d50ad8019ef2426039307ff5dfbe70c736ad0d4d401facf304", [:mix], [], "hexpm"}, + "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recase": {:hex, :recase, "0.4.0", "8fb52846f75948156385af2dfdc12f69e5ce27b022a9d1682c70a2fb3ed149c7", [:mix], [], "hexpm"}, "rihanna": {:hex, :rihanna, "1.3.5", "5f5e6c5b1e514978a29a6791f338f4bb963401959fc212bd18d4a2c92d79a7a4", [:mix], [{:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.13.3", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm"}, @@ -66,6 +71,7 @@ "short_maps": {:hex, :short_maps, "0.1.2", "a7c2bfd91179cdbdfe90e74a023992335d116982fa672612c74776b2e9257a7b", [:mix], [], "hexpm"}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, + "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "0.10.0", "e588c7e7f1c0866c81eeed5c38f02a4a94d6309eede336c1e6ca08b0a95abd3f", [:mix], [{:exjsx, ">= 0.1.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/groupher_server/statistics/statistics_test.exs b/test/groupher_server/statistics/statistics_test.exs index 63c831179..a767be14e 100644 --- a/test/groupher_server/statistics/statistics_test.exs +++ b/test/groupher_server/statistics/statistics_test.exs @@ -121,7 +121,6 @@ defmodule GroupherServer.Test.Statistics do assert second.count == 2 end - @tag :wip test "should return recent #{@community_contribute_days} days community contributes by default", ~m(community)a do days_ago = Timex.shift(Timex.today(), days: -@community_contribute_days) @@ -148,7 +147,6 @@ defmodule GroupherServer.Test.Statistics do assert length(contributes) == @community_contribute_days + 1 end - @tag :wip test "the contributes data should be cached after first query", ~m(community)a do scope = Cache.get_scope(:community_contributes, community.id) assert {:error, nil} = Cache.get(scope) @@ -158,7 +156,6 @@ defmodule GroupherServer.Test.Statistics do assert {:ok, contributes} = Cache.get(scope) end - @tag :wip2 test "Rihanna should work in test sandbox", ~m(community)a do res = Rihanna.enqueue({IO, :puts, ["Work, work, work, work, work."]}) Process.sleep(1000) diff --git a/test/helper/cache_test.exs b/test/helper/cache_test.exs index 6486653ec..904813f83 100644 --- a/test/helper/cache_test.exs +++ b/test/helper/cache_test.exs @@ -5,13 +5,11 @@ defmodule GroupherServer.Test.Helper.Cache do alias Helper.Cache describe "[cache test]" do - @tag :wip test "cache get unexsit key should get nil" do assert {:error, nil} = Cache.get("no exsit") assert {:error, nil} = Cache.get(:no_exsit) end - @tag :wip test "cache put should work" do assert {:error, nil} = Cache.get(:data) @@ -26,5 +24,22 @@ defmodule GroupherServer.Test.Helper.Cache do assert {:ok, true} = Cache.put("namespace.aaa.bbb", [1, %{a: "2"}]) assert {:ok, [1, %{a: "2"}]} = Cache.get("namespace.aaa.bbb") end + + test "cache can be clear" do + assert {:ok, true} = Cache.put(:data, "value") + assert {:ok, "value"} = Cache.get(:data) + + assert {:ok, _} = Cache.clear_all() + assert {:error, nil} = Cache.get(:data) + end + + test "cache expire should work" do + assert {:ok, true} = Cache.put(:data, "value", expire: 1000) + assert {:ok, "value"} = Cache.get(:data) + Process.sleep(900) + assert {:ok, "value"} = Cache.get(:data) + Process.sleep(1200) + assert {:error, nil} = Cache.get(:data) + end end end