From 9460056e7a168242195e3190e7ec37b99ef7e827 Mon Sep 17 00:00:00 2001 From: Natsu Kagami Date: Sat, 18 Jan 2020 18:00:25 -0500 Subject: [PATCH] Parallelize announcer message-sending --- Cargo.lock | 85 ++++++++++++++++++++++++++ youmubot/Cargo.toml | 1 + youmubot/src/commands/announcer.rs | 10 ++- youmubot/src/commands/osu/announcer.rs | 43 ++++++++----- 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 281a3d4..a0b61f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,6 +183,47 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crossbeam-deque" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ct-logs" version = "0.6.0" @@ -209,6 +250,11 @@ name = "dtoa" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "encoding_rs" version = "0.8.22" @@ -579,6 +625,14 @@ name = "memchr" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "memoffset" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mime" version = "0.3.16" @@ -862,6 +916,28 @@ dependencies = [ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rayon" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rayon-core" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "redox_syscall" version = "0.1.56" @@ -1605,6 +1681,7 @@ dependencies = [ "dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustbreak 2.0.0-rc3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1653,10 +1730,15 @@ dependencies = [ "checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" "checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" +"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" +"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" "checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" +"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28" "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" @@ -1701,6 +1783,7 @@ dependencies = [ "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" "checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" "checksum mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" "checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625" @@ -1733,6 +1816,8 @@ dependencies = [ "checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" "checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" "checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +"checksum rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" +"checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5508c1941e4e7cb19965abef075d35a9a8b5cdf0846f30b4050e9b55dc55e87" "checksum regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e734e891f5b408a29efbf8309e656876276f49ab6a6ac208600b4419bd893d90" diff --git a/youmubot/Cargo.toml b/youmubot/Cargo.toml index d4cfdad..5df5fbb 100644 --- a/youmubot/Cargo.toml +++ b/youmubot/Cargo.toml @@ -17,6 +17,7 @@ reqwest = "0.10.1" regex = "1" lazy_static = "1" youmubot-osu = { path = "../youmubot-osu" } +rayon = "1.1" [dependencies.rustbreak] version = "2.0.0-rc3" diff --git a/youmubot/src/commands/announcer.rs b/youmubot/src/commands/announcer.rs index 73557b0..dd25ecd 100644 --- a/youmubot/src/commands/announcer.rs +++ b/youmubot/src/commands/announcer.rs @@ -15,7 +15,7 @@ pub trait Announcer { fn send_messages( c: &Http, d: &mut ShareMap, - channels: impl Fn(UserId) -> Vec, + channels: impl Fn(UserId) -> Vec + Sync, ) -> CommandResult; fn set_channel(d: &mut ShareMap, guild: GuildId, channel: ChannelId) -> CommandResult { @@ -48,8 +48,12 @@ pub trait Announcer { let mut v = Vec::with_capacity(guilds.len()); for (guild, channel) in guilds.into_iter() { let mut s = HashSet::new(); - for user in guild.members_iter(c.as_ref()) { - s.insert(user?.user_id()); + for user in guild + .members_iter(c.as_ref()) + .take_while(|u| u.is_ok()) + .filter_map(|u| u.ok()) + { + s.insert(user.user_id()); } v.push((s, channel)) } diff --git a/youmubot/src/commands/osu/announcer.rs b/youmubot/src/commands/osu/announcer.rs index a39fce0..60a8166 100644 --- a/youmubot/src/commands/osu/announcer.rs +++ b/youmubot/src/commands/osu/announcer.rs @@ -4,6 +4,7 @@ use crate::{ db::{OsuSavedUsers, OsuUser}, http::{Osu, HTTP}, }; +use rayon::prelude::*; use reqwest::blocking::Client as HTTPClient; use serenity::{ framework::standard::{CommandError as Error, CommandResult}, @@ -30,7 +31,7 @@ impl Announcer for OsuAnnouncer { fn send_messages( c: &Http, d: &mut ShareMap, - channels: impl Fn(UserId) -> Vec, + channels: impl Fn(UserId) -> Vec + Sync, ) -> CommandResult { let http = d.get::().expect("HTTP"); let osu = d.get::().expect("osu!client"); @@ -51,21 +52,31 @@ impl Announcer for OsuAnnouncer { .unwrap() .unwrap() }); - for (rank, score) in scores { - let beatmap = BeatmapWithMode( - osu.beatmaps(http, BeatmapRequestKind::Beatmap(score.beatmap_id), |f| f)? - .into_iter() - .next() - .unwrap(), - *mode, - ); - for channel in channels(*user_id) { - channel.send_message(c, |c| { - c.content(format!("New top record from {}!", user_id.mention())) - .embed(|e| score_embed(&score, &beatmap, &user, Some(rank), e)) - })?; - } - } + scores + .into_par_iter() + .filter_map(|(rank, score)| { + let beatmap = osu + .beatmaps(http, BeatmapRequestKind::Beatmap(score.beatmap_id), |f| f) + .map(|v| BeatmapWithMode(v.into_iter().next().unwrap(), *mode)); + let channels = channels(*user_id); + match beatmap { + Ok(v) => Some((rank, score, v, channels)), + Err(e) => { + dbg!(e); + None + } + } + }) + .for_each(|(rank, score, beatmap, channels)| { + for channel in channels { + if let Err(e) = channel.send_message(c, |c| { + c.content(format!("New top record from {}!", user_id.mention())) + .embed(|e| score_embed(&score, &beatmap, &user, Some(rank), e)) + }) { + dbg!(e); + } + } + }); } osu_user.last_update = chrono::Utc::now(); }