Fix CI

Add k8s deploy example

.github/workflows/automerge.yml

@@ -0,0 +1,27 @@
+name: automerge
+on:
+  pull_request:
+    types:
+      - labeled
+      - unlabeled
+      - synchronize
+      - opened
+      - edited
+      - ready_for_review
+      - reopened
+      - unlocked
+  pull_request_review:
+    types:
+      - submitted
+  check_suite:
+    types:
+      - completed
+  status: {}
+jobs:
+  automerge:
+    runs-on: ubuntu-latest
+    steps:
+      - name: automerge
+        uses: "pascalgn/automerge-action@v0.13.0"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/build-push.yml

@@ -31,6 +31,41 @@ jobs:
         with:
           name: desubot.exe
           path: ./target/release/desubot.exe
+  build-push-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Branch name
+        id: branch_name
+        run: |
+          echo ::set-output name=SOURCE_NAME::${GITHUB_REF#refs/*/}
+          echo ::set-output name=SOURCE_BRANCH::${GITHUB_REF#refs/heads/}
+          echo ::set-output name=SOURCE_TAG::${GITHUB_REF#refs/tags/}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        id: docker_build_latest
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ultradesu/desubot:latest
+      -
+        name: Build and push
+        id: docker_build_tag
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ultradesu/desubot:${{ steps.branch_name.outputs.SOURCE_TAG }}
 
   publish:
     name: Publish release

.gitignore

@@ -6,4 +6,5 @@ memory.sqlite3
 /video
 /voice
 /.idea
-Cargo.lock
+Cargo.lock
+k8s/k8s-deploy.yaml

Cargo.toml

@@ -1,41 +1,42 @@
-[package]
+[package]
-name = "desubot"
+name = "desubot"
-version = "0.5.1"
+version = "0.5.7"
-authors = ["AB <ab@hexor.ru>"]
+authors = ["AB <ab@hexor.ru>"]
-edition = "2018"
+edition = "2018"
-
+
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
+
-[dependencies]
+[dependencies]
-bytes = "0.5"
+bytes = "0.5"
-tokio = { version = "0.2", features = ["full"]}
+tokio = { version = "0.2", features = ["full"]}
-tracing = "0.1.9"
+tracing = "0.1.9"
-tracing-futures = "0.2"
+tracing-futures = "0.2"
-multipart = { version = "0.16", default-features = false, features = ["client"] }
+multipart = { version = "0.16", default-features = false, features = ["client"] }
-telegram-bot = "0.8.0"
+#telegram-bot = "0.8.0"
-silicon = "0.4.0"
+telegram-bot = { git = "https://github.com/ayrat555/telegram-bot", branch = "ayrat555/api-fixes-10" }
-hyper = "0.13"
+silicon = "0.4.0"
-hyper-tls = { version = "0.4", optional = true  }
+hyper = "0.13"
-futures = "0.3"
+hyper-tls = { version = "0.4", optional = true  }
-hyper-rustls = { version = "0.19", optional = true }
+futures = "0.3"
-rusqlite = { version = "0.24.2", features = ["bundled"]}
+hyper-rustls = { version = "0.19", optional = true }
-html-escape = "0.2"
+rusqlite = { version = "0.24.2", features = ["bundled"]}
-regex = "1"
+html-escape = "0.2"
-reqwest = "0.10.9"
+regex = "1"
-uuid = { version = "0.8", features = ["v4"] }
+reqwest = "0.10.9"
-sha1 = "0.6.0"
+uuid = { version = "0.8", features = ["v4"] }
-env_logger = "0.7"
+sha1 = "0.6.0"
-log = { version = "^0.4.5", features = ["std"] }
+env_logger = "0.7"
-subprocess = "0.2.6"
+log = { version = "^0.4.5", features = ["std"] }
-serde_json = "1.0"
+subprocess = "0.2.6"
-markov = "1.1.0"
+serde_json = "1.0"
-rand = "0.7.3"
+markov = "1.1.0"
-mystem = "^0.2"
+rand = "0.7.3"
-#mystem = { path = "../mystem-rs" }
+mystem = "^0.2"
-async-trait = "0.1.42"
+#mystem = { path = "../mystem-rs" }
-sqlparser = "0.7.0"
+async-trait = "0.1.42"
-
+sqlparser = "0.7.0"
-[dependencies.syntect]
+
-version = "4.4"
+[dependencies.syntect]
-default-features = false
+version = "4.4"
-features = ["parsing", "dump-load", "regex-onig"]
+default-features = false
+features = ["parsing", "dump-load", "regex-onig"]

Dockerfile

@@ -0,0 +1,14 @@
+# syntax=docker/dockerfile:1
+
+FROM rust:latest AS builder
+WORKDIR /desubot
+ADD ./ /desubot/
+RUN cargo build --release
+
+FROM ubuntu:latest
+WORKDIR /storage
+COPY --from=builder /desubot/target/release/desubot /usr/bin/
+COPY mystem /usr/bin/
+RUN apt update && apt install -y fontconfig openssl ca-certificates && rm -rf /var/lib/apt/lists/*
+ENTRYPOINT desubot
+

README

@@ -1,3 +1,5 @@
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fhouse-of-vanity%2Fdesubot.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fhouse-of-vanity%2Fdesubot?ref=badge_shield)
+
 Desubot
 Telegram bot with light group statistic and heavy spy features.
 
@@ -7,10 +9,12 @@ Telegram bot with light group statistic and heavy spy features.
 * /here command to mention all members.
 * Alongside with saving whole message bot perform blacklist filter and stemming for every word (only Russian). "Красивую собаку мыли негры" -> "красивый собака мыть негр"
 * Generate sentences using Markov Chains trained on history with /markov_all.
-
-== TODO ==
 * Syntax highlighting for CODE exported to image.
 
 == Important ==
 * Desubot uses MyStem by Yandex for word stemming and assume that mystem binary is available in PATH.
-On Windows it may be placed on working directory. Both Linux and Windows mystem binary is in repo.
+On Windows it may be placed on working directory. Both Linux and Windows mystem binary is in repo.
+
+
+## License
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fhouse-of-vanity%2Fdesubot.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fhouse-of-vanity%2Fdesubot?ref=badge_large)

assets/help_text.rs

@@ -2,7 +2,7 @@
 static CODE_HELP: &str = "<b>Code highlighter</b>
 
 <i>Usage</i>
-<pre>/CODE
+<pre>/code
 &lt;CODE&gt;
 #&lt;lang - JS by default&gt; #&lt;theme - Dracula by default&gt;</pre>
 

k8s/k8s-deploy-example.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: desubot-api-token
+data:
+  token: 123.... # Base64 encoded token.
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: desubot
+spec:
+  serviceName: "desubot"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: desubot
+  template:
+    metadata:
+      labels:
+        app: desubot
+    spec:
+      containers:
+      - name: desubot
+        image: ultradesu/desubot:latest
+        volumeMounts:
+        - name: storage
+          mountPath: /storage
+        env:
+        - name: TELEGRAM_BOT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: desubot-api-token
+              key: token
+  volumeClaimTemplates:
+  - metadata:
+      name: storage
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 50Gi

src/commands.rs

@@ -21,7 +21,7 @@ use syntect::highlighting::Theme;
 use syntect::parsing::SyntaxReference;
 use syntect::util::LinesWithEndings;
 use telegram_bot::prelude::*;
-use telegram_bot::{Api, Message, ParseMode};
+use telegram_bot::{Api, Message, ParseMode, UserId};
 
 include!("../assets/help_text.rs");
 
@@ -69,7 +69,7 @@ impl Execute for Sql {
         let mut sql = self.data.clone();
         debug!("PIZDA - {}", sql);
         if sql == "/sql" || sql == "/sql-" {
-            return Ok(SQL_HELP.to_string())
+            return Ok(SQL_HELP.to_string());
         }
         let is_head = if sql.starts_with('-') {
             sql = sql.replacen("-", "", 1);
@@ -143,7 +143,7 @@ impl Execute for Sql {
             }
             res.push(tmp);
         }
-        if res.len() > 100 {
+        if res.len() >= 100 {
             return Err(Error::SQLResultTooLong(
                 "SQL result too long. Lines limit is 100. Use LIMIT".to_string(),
             ));
@@ -192,7 +192,15 @@ impl Execute for Sql {
 #[async_trait]
 impl Execute for Here {
     async fn exec(&self, api: &Api, message: &Message) -> Result<(), Error> {
-        let members: Vec<telegram_bot::User> = db::get_members(message.chat.id()).unwrap();
+        let members: Vec<telegram_bot::User> =
+            db::get_members(message.chat.id(), 60).unwrap_or(vec![telegram_bot::User {
+                id: UserId::new(124317807),
+                first_name: "Ultradesu".to_string(),
+                last_name: None,
+                username: None,
+                is_bot: false,
+                language_code: None,
+            }]);
         for u in &members {
             debug!("Found user {:?} in chat {}", u, message.chat.id());
         }

src/db.rs

@@ -180,18 +180,29 @@ pub(crate) async fn get_messages_user_all(
     Ok(messages)
 }
 
-pub(crate) fn get_members(id: telegram_bot::ChatId) -> Result<Vec<telegram_bot::User>> {
+pub(crate) fn get_members(id: telegram_bot::ChatId, limit: u32) -> Result<Vec<telegram_bot::User>> {
+    let where_statement = if limit > 0 {
+        format!("and days_seen <= {}", limit)
+    } else {
+        "".into()
+    };
+    debug!("{}", where_statement);
     let conn = open()?;
-    let mut stmt = conn.prepare_cached(
+    let mut stmt = conn.prepare_cached(&format!(
         "
-        SELECT DISTINCT(u.username), u.id, u.first_name, u.last_name, u.date
+        SELECT DISTINCT(u.username), u.id, u.first_name, u.last_name, u.date,
+        (strftime('%s','now')-r.date)/60/60/24 as days_seen
         FROM relations r
         JOIN user u
         ON u.id = r.user_id
         LEFT JOIN conf c
         ON r.conf_id = c.id
-        WHERE c.id = :id",
+        WHERE c.id = :id
-    )?;
+        {}
+        GROUP BY u.id
+        ORDER BY r.date DESC",
+        where_statement
+    ))?;
     let mut rows = stmt.query_named(&[(":id", &id.to_string())])?;
     let mut users = Vec::new();
 
@@ -214,7 +225,7 @@ pub(crate) async fn add_conf(message: Message) -> Result<(), Error> {
 
     match get_conf(message.chat.id()) {
         Ok(_) => {
-            //info!("Group found: {:?}", message.chat.id());
+            debug!("Group found: {:?}", message.chat.id());
             let update = Conf {
                 id: message.chat.id(),
                 title,
@@ -228,10 +239,10 @@ pub(crate) async fn add_conf(message: Message) -> Result<(), Error> {
                 id = :id",
             )?;
             stmt.execute_named(&[(":id", &update.id.to_string()), (":title", &update.title)])?;
-            //info!("Conf {:?} updated: {:?}", update.title, get_conf(update.id));
+            debug!("Conf {:?} updated: {:?}", update.title, get_conf(update.id));
         }
         Err(_) => {
-            //info!("Group didn't found: {:?}", message.chat.id());
+            debug!("Group didn't found: {:?}", message.chat.id());
 
             let update = Conf {
                 id: message.chat.id(),
@@ -250,7 +261,7 @@ pub(crate) async fn add_conf(message: Message) -> Result<(), Error> {
                 (":title", &update.title),
                 (":date", &unix_time),
             ])?;
-            //info!("Conf {:?} added: {:?}", update.title, get_conf(update.id));
+            debug!("Conf {:?} added: {:?}", update.title, get_conf(update.id));
         }
     }
     Ok(())
@@ -283,7 +294,11 @@ pub(crate) async fn add_user(message: Message) -> Result<(), Error> {
                 (":first_name", &update.first_name),
                 (":last_name", &update.last_name),
             ])?;
-            //println!("User {} updated: {:?}", update.first_name, get_user(user.id));
+            debug!(
+                "User {} updated: {:?}",
+                update.first_name,
+                get_user(update.id)
+            );
         }
         Err(_) => {
             let unix_time = SystemTime::now()
@@ -310,7 +325,7 @@ pub(crate) async fn add_user(message: Message) -> Result<(), Error> {
                 (":last_name", &user.last_name),
                 (":date", &unix_time),
             ])?;
-            //println!("User added: {:?}", user);
+            debug!("User added: {:?}", user);
         }
     }
     Ok(())
@@ -417,6 +432,7 @@ pub(crate) async fn add_sentence(
         };
 
     // Save stemmed words
+    debug!("Going to stem: {}", text);
     let words = mystem.stemming(text)?;
     conn.execute("BEGIN TRANSACTION", params![]);
     for word in words {

src/handlers.rs

@@ -53,7 +53,12 @@ pub async fn handler(
                         }
                     }
                 }
-                s if s.contains("/here") => {
+                s if s.contains("/here")
+                    || s.contains("@here")
+                    || s.contains("/хере")
+                    || s.contains("@хере")
+                    || s.contains("\"хере") =>
+                {
                     db::add_sentence(&message, mystem).await?;
                     Here {
                         data: "".to_string(),
@@ -118,9 +123,7 @@ pub async fn handler(
                     .exec_mystem(&api, &message, mystem)
                     .await?
                 }
-                _ => {
+                _ => db::add_sentence(&message, mystem).await?,
-                    db::add_sentence(&message, mystem).await?
-                }
             }
         }
         MessageKind::Photo { ref caption, .. } => {

src/main.rs

@@ -1,4 +1,6 @@
+#![allow(unreachable_code)]
 use std::{env, process};
+use tokio::time::{delay_for, Duration};
 
 use futures::StreamExt;
 use telegram_bot::*;
@@ -16,7 +18,7 @@ use mystem::MyStem;
 
 #[tokio::main]
 async fn main() -> Result<(), errors::Error> {
-    env_logger::from_env(Env::default().default_filter_or("info")).init();
+    env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
     let mut mystem = match MyStem::new() {
         Ok(mystem) => mystem,
         Err(e) => {
@@ -44,18 +46,33 @@ async fn main() -> Result<(), errors::Error> {
         me.first_name,
         me.id
     );
-    while let Some(update) = stream.next().await {
+    loop {
-        let update = update?;
+        while let Some(update) = stream.next().await {
-        if let UpdateKind::Message(message) = update.kind {
+            match update {
-            db::add_conf(message.clone()).await?;
+                Ok(u) => {
-            db::add_user(message.clone()).await?;
+                    if let UpdateKind::Message(message) = u.kind {
-            match handlers::handler(api.clone(), message, token.clone(), &mut mystem, me.clone())
+                        db::add_conf(message.clone()).await?;
-                .await
+                        db::add_user(message.clone()).await?;
-            {
+                        match handlers::handler(
-                Ok(_) => {}
+                            api.clone(),
-                Err(e) => warn!("An error occurred handling command. {:?}", e),
+                            message,
-            }
+                            token.clone(),
+                            &mut mystem,
+                            me.clone(),
+                        )
+                        .await
+                        {
+                            Ok(_) => {}
+                            Err(e) => warn!("An error occurred handling command. {:?}", e),
+                        }
+                    }
+                }
+                Err(e) => {
+                    warn!("Telegram API Error: {:?}", e);
+                }
+            };
         }
+        delay_for(Duration::from_secs(2)).await;
     }
     Ok(())
 }