User overview page implemented.

database.py

@@ -1,8 +1,10 @@
+import sqlite3
+from datetime import datetime
+from dateutil import parser
+import logging
+
 class DataBase:
     def __init__(self, basefile):
-        import sqlite3
-        #import datetime as dt
-        import logging
         self.log = logging.getLogger("pycrm." + __name__)
         try:
             self.conn = sqlite3.connect(
@@ -29,7 +31,28 @@ class DataBase:
         sql = "SELECT * FROM `meme` ORDER BY rowid DESC "
         return(self.execute(sql))
 
+    def get_user_count(self):
+        sql = "SELECT count(*) FROM `user`"
+        return(self.execute(sql)[0])
+
+    def get_word_count(self):
+        sql = "SELECT count(*) FROM `word`"
+        return(self.execute(sql)[0])
+
+    def get_relations_count(self):
+        sql = "SELECT count(*) FROM `relations`"
+        return(self.execute(sql)[0])
+
+    def get_confs_count(self):
+        sql = "SELECT count(*) FROM `conf`"
+        return(self.execute(sql)[0])
+
     def get_users(self, order='id', sorting='ASC'):
+        # sql injection prevention
+        if sorting == 'ASC':
+            sorting = 'ASC'
+        else:
+            sorting = 'DESC'
         if order == 'id':
             order = 'id'
         elif order == 'first_name':
@@ -42,23 +65,99 @@ class DataBase:
             order = 'dt'
         elif order == 'last_activity':
             order = 'last_seen'
+        elif order == 'count':
+            order = 'count'
         else:
             order = 'id'
         sql = """
-        SELECT * FROM 
-            (SELECT u.id,
+        SELECT * FROM (
+            SELECT u.id,
             u.username,
             u.first_name,
             u.last_name,
-            datetime(u.date, 'unixepoch') as dt,
-            max(datetime(r.date, 'unixepoch')) as last_seen
+            date(u.date, 'unixepoch') as dt,
+            max(date(r.date, 'unixepoch')) as last_seen,
+            count(u.id) as count
             FROM `user` u LEFT JOIN `relations` r ON
             r.user_id == u.id
-            GROUP BY u.id)
+            GROUP BY u.id
+        )
          ORDER BY %s %s""" % (
             order, sorting)
-
         return(self.execute(sql))
 
+    def get_confs(self):
+        sql = """
+            SELECT c.title,
+            c.id,
+            date(c.date, 'unixepoch') as dt,
+            count(r.conf_id) as count,
+            t1.users
+            FROM `conf` c 
+            LEFT JOIN `relations` r 
+            ON c.id = r.conf_id
+            LEFT JOIN `user` u
+            ON u.id = r.user_id
+            LEFT JOIN (
+                SELECT id, title, count(user_id) as users FROM (
+                SELECT c.id, c.title, r.user_id, count(r.conf_id) as words
+                FROM `conf` c 
+                LEFT JOIN `relations` r 
+                ON c.id = r.conf_id
+                GROUP BY c.id, r.user_id
+            )
+            GROUP BY title) as t1
+            ON t1.id = c.id
+            GROUP BY c.id
+        """
+        return(self.execute(sql))
+
+    def get_user_info(self, user_id):
+        if not user_id.isdigit():
+            return False
+        raw1 = self.execute("""
+            SELECT u.id,
+            first_name,
+            last_name,
+            username,
+            datetime(u.date, 'unixepoch') as date,
+            count(u.id) as words,
+            datetime(max(r.date), 'unixepoch') as last_message
+            FROM `user` u
+            LEFT JOIN `relations` r ON r.user_id = u.id
+            WHERE u.id = %s""" % user_id)[0]
+        top = self.execute("""
+            SELECT w.word, count(w.id) as count FROM `relations` r 
+            LEFT JOIN `user` u ON u.id = r.user_id
+            LEFT JOIN `word` w ON w.id = r.word_id
+            WHERE u.id = %s
+            GROUP BY w.id
+            ORDER BY count DESC
+            LIMIT 10
+            """ % user_id)
+        chats = self.execute("""SELECT c.title,
+            count(c.id) count,
+            min(date(r.date, 'unixepoch'))
+            FROM `relations` r 
+            LEFT JOIN `user` u ON u.id = r.user_id
+            LEFT JOIN `conf` c ON c.id = r.conf_id
+            WHERE u.id = %s
+            GROUP BY c.id""" % user_id)
+
+        day_known = (parser.parse(raw1[6]) - parser.parse(raw1[4])).days
+        user_info = {
+            'id': raw1[0],
+            'first_name': raw1[1],
+            'last_name': raw1[2],
+            'username': raw1[3],
+            'first_date': raw1[4],
+            'word_count': raw1[5],
+            'last_message': raw1[6],
+            'day_known': day_known,
+            'top': top,
+            'chats': chats,
+        }
+        return user_info
+
     def close(self):
         self.conn.close()

index.py

@@ -35,11 +35,46 @@ def serve_static(path):
 def index():
     order = request.args.get('order', default = 'id', type = str)
     sorting = request.args.get('sorting', default = 'ASC', type = str)
-    return render_template('index.html', users=db.get_users(order=order, sorting=sorting), sorting=sorting)
+    totals = {
+        'users': db.get_user_count(),
+        'words': db.get_word_count(),
+        'relations': db.get_relations_count(),
+        'confs': db.get_confs_count()
+    }
+    return render_template(
+        'index.html',
+        users=db.get_users(order=order, sorting=sorting),
+        sorting=sorting,
+        totals=totals
+        )
 
-@app.route('/users_overview')
-def users_overview():
-    return render_template('users_overview.html')
+@app.route('/conf')
+def conf():
+    totals = {
+        'users': db.get_user_count(),
+        'words': db.get_word_count(),
+        'relations': db.get_relations_count(),
+        'confs': db.get_confs_count()
+    }
+
+    return render_template(
+        'conf.html',
+        confs=db.get_confs(),
+        totals=totals)
+
+@app.route('/overview/user/<user_id>')
+def user_overview(user_id):
+    totals = {
+        'users': db.get_user_count(),
+        'words': db.get_word_count(),
+        'relations': db.get_relations_count(),
+        'confs': db.get_confs_count()
+    }
+
+    return render_template(
+        'user.html',
+        user_info=db.get_user_info(user_id),
+        totals=totals)
 
 
 def main():

templates/conf.html

@@ -0,0 +1,83 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+  <!-- Required meta tags -->
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
+  <!-- Bootstrap CSS -->
+  <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO"
+    crossorigin="anonymous">
+
+  <title>libOpenAnal appliance</title>
+</head>
+
+<body>
+  <div class="container">
+
+    <h1>Hexor's conf_bot data extractor tool</h1>
+    <ul class="nav">
+      <li class="nav-item">
+        <a class="nav-link active" href="/">Users</a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" href="/conf">Conferences</a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" href="#">Link</a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link disabled" href="#">Disabled</a>
+      </li>
+    </ul>
+
+
+    <div class="row">
+      <div class="col-sm-12">
+        <h4>Totals</h4>
+        <b>Users: </b> {{ totals.users[0] }}<br>
+        <b>Unique words: </b> {{ totals.words[0] }}<br>
+        <b>Words said: </b> {{ totals.relations[0] }}<br>
+        <b>Chats fetched: </b> {{ totals.confs[0] }}<br>
+      </div>
+    </div>
+
+    <table class="table table-hover table-sm">
+      <thead>
+        <tr>
+          <th scope="col">#</th>
+          <th scope="col">Title</th>
+          <th scope="col">ID</th>
+          <th scope="col">Added</th>
+          <th scope="col">Members</th>
+          <th scope="col">Words said</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for conf in confs %}
+        <tr>
+          <th scope="row">{{loop.index}}</th>
+          <td>{{ conf.0 }}</td>
+          <td>{{ conf.1 }}</td>
+          <td>{{ conf.2 }}</td>
+          <td>{{ conf.4 }}</td>
+          <td>{{ conf.3 }}</td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+    <!-- Optional JavaScript -->
+    <!-- jQuery first, then Popper.js, then Bootstrap JS -->
+    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
+      crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
+      crossorigin="anonymous"></script>
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
+      crossorigin="anonymous"></script>
+  </div>
+  </div>
+  </div>
+</body>
+
+</html>

templates/index.html

@@ -14,60 +14,78 @@
 </head>
 
 <body>
-  <h1>Hexor's conf_bot data extractor tool</h1>
+  <div class="container">
 
-  <ul class="nav">
-    <li class="nav-item">
-      <a class="nav-link active" href="#">Users overview</a>
-    </li>
-    <li class="nav-item">
-      <a class="nav-link" href="#">Link</a>
-    </li>
-    <li class="nav-item">
-      <a class="nav-link" href="#">Link</a>
-    </li>
-    <li class="nav-item">
-      <a class="nav-link disabled" href="#">Disabled</a>
-    </li>
-  </ul>
+    <h1>Hexor's conf_bot data extractor tool</h1>
+    <ul class="nav">
+      <li class="nav-item">
+        <a class="nav-link active" href="/">Users</a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" href="/conf">Conferences</a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" href="#">Link</a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link disabled" href="#">Disabled</a>
+      </li>
+    </ul>
+    <div class="row">
+      <div class="col-sm-12">
+        <h4>Totals</h4>
+        <b>Users: </b> {{ totals.users[0] }}<br>
+        <b>Unique words: </b> {{ totals.words[0] }}<br>
+        <b>Words said: </b> {{ totals.relations[0] }}<br>
+        <b>Chats fetched: </b> {{ totals.confs[0] }}<br>
+      </div>
+    </div>
 
-
-
-  <table class="table table-hover table-sm">
-    <thead>
-      <tr>
-        <th scope="col">#</th>
-        <th scope="col"><a href="./?order=first_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">First
-            name</a></th>
-        <th scope="col"><a href="./?order=last_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last name</a></th>
-        <th scope="col"><a href="./?order=username&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Username</a></th>
-        <th scope="col"><a href="./?order=id&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">User ID</a></th>
-        <th scope="col"><a href="./?order=firstly_seen&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Firstly seen</a></th>
-        <th scope="col"><a href="./?order=last_activity&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last activity</a></th>
-      </tr>
-    </thead>
-    <tbody>
-      {% for user in users %}
-      <tr>
-        <th scope="row">{{loop.index}}</th>
-        <td>{{ user.2 }}</td>
-        <td>{{ user.3 }}</td>
-        <td>{{ user.1 }}</td>
-        <td>{{ user.0 }}</td>
-        <td>{{ user.4 }}</td>
-        <td>{{ user.5 }}</td>
-      </tr>
-      {% endfor %}
-    </tbody>
-  </table>
-  <!-- Optional JavaScript -->
-  <!-- jQuery first, then Popper.js, then Bootstrap JS -->
-  <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
-    crossorigin="anonymous"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
-    crossorigin="anonymous"></script>
-  <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
-    crossorigin="anonymous"></script>
+    <table class="table table-hover table-sm">
+      <thead>
+        <tr>
+          <th scope="col">#</th>
+          <th scope="col"><a href="./?order=first_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">First
+              name</a></th>
+          <th scope="col"><a href="./?order=last_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last
+              name</a></th>
+          <th scope="col"><a href="./?order=username&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Username</a></th>
+          <th scope="col"><a href="./?order=id&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">User
+              ID</a></th>
+          <th scope="col"><a href="./?order=firstly_seen&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Firstly
+              seen</a></th>
+          <th scope="col"><a href="./?order=last_activity&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last
+              activity</a></th>
+          <th scope="col"><a href="./?order=count&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Word
+              count</a></th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for user in users %}
+        <tr>
+          <th scope="row">{{loop.index}}</th>
+          <td><a href="/overview/user/{{ user.0 }}">{{ user.2 }}</a></td>
+          <td>{{ user.3 }}</td>
+          <td>{{ user.1 }}</td>
+          <td><a class="badge badge-dark " href="/overview/user/{{ user.0 }}">{{ user.0 }}</a></td>
+          <td>{{ user.4 }}</td>
+          <td>{{ user.5 }}</td>
+          <td>{{ user.6 }}</td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+    <!-- Optional JavaScript -->
+    <!-- jQuery first, then Popper.js, then Bootstrap JS -->
+    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
+      crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
+      crossorigin="anonymous"></script>
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
+      crossorigin="anonymous"></script>
+  </div>
+  </div>
+  </div>
 </body>
 
 </html>

templates/user.html

@@ -0,0 +1,131 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+    <!-- Required meta tags -->
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
+    <!-- Bootstrap CSS -->
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO"
+        crossorigin="anonymous">
+
+    <title>libOpenAnal appliance</title>
+</head>
+
+<body>
+    <div class="container">
+
+        <h1>Hexor's conf_bot data extractor tool</h1>
+        <ul class="nav">
+            <li class="nav-item">
+                <a class="nav-link active" href="/">Users</a>
+            </li>
+            <li class="nav-item">
+                <a class="nav-link" href="/conf">Conferences</a>
+            </li>
+            <li class="nav-item">
+                <a class="nav-link" href="#">Link</a>
+            </li>
+            <li class="nav-item">
+                <a class="nav-link disabled" href="#">Disabled</a>
+            </li>
+        </ul>
+        <hr>
+        <h4>{{ user_info.username }} user's data</h4>
+        <hr>
+        <div class="card-columns">
+            <div class="card">
+                <div class="card-body">
+                    <h5 class="card-title">Identity</h5>
+                    <hr>
+                    <p class="card-text">
+                        <b>First name: </b>{{ user_info.first_name }}<br>
+                        <b>Last name: </b>{{user_info.last_name }}<br>
+                        <b>Username: </b>{{user_info.username }}<br>
+                        <b>Telegram ID: </b>{{user_info.id }}<br>
+                        <b>Firstly seen: </b>{{user_info.first_date }}<br>
+                    </p>
+                </div>
+            </div>
+            <div class="card">
+                <div class="card-body">
+                    <h5 class="card-title">Activity</h5>
+                    <hr>
+                    <p class="card-text">
+                        <b>First message: </b>{{user_info.first_date }}<br>
+                        <b>Last message: </b>{{user_info.last_message }}<br>
+                        <b>Days known: </b>{{user_info.day_known }}<br>
+                        <b>Word said: </b>{{ user_info.word_count }}<br>
+                        <b>Words per day: </b>{{ (user_info.word_count / user_info.day_known)|int }}<br>
+                    </p>
+                </div>
+            </div>
+            <div class="card">
+                <div class="card-body">
+                    <h5 class="card-title">Top</h5>
+                    <hr>
+                    <p class="card-text">
+                        <table class="table table-hover table-sm">
+                            <thead>
+                                <tr>
+                                    <th scope="col">#</th>
+                                    <th scope="col">Word</th>
+                                    <th scope="col">Said</th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for word in user_info.top %}
+                                <tr>
+                                    <th scope="row">{{loop.index}}</th>
+                                    <td>{{word[0]}}</td>
+                                    <td><span class="badge badge-secondary">{{word[1]}} </span></td>
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
+                    </p>
+                </div>
+            </div>
+            <div class="card">
+                <div class="card-body">
+                    <h5 class="card-title">Chats</h5>
+                    <p class="card-text">
+                            <table class="table table-hover table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">#</th>
+                                        <th scope="col">Chat</th>
+                                        <th scope="col">Words said</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for chat in user_info.chats %}
+                                    <tr>
+                                        <th scope="row">{{loop.index}}</th>
+                                        <td>{{chat[0]}}</td>
+                                        <td><span class="badge badge-secondary">{{chat[1]}} </span></td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </p>
+                    </div>
+            </div>
+        </div>
+
+
+        <!-- Optional JavaScript -->
+        <!-- jQuery first, then Popper.js, then Bootstrap JS -->
+        <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
+            crossorigin="anonymous"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
+            crossorigin="anonymous"></script>
+        <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
+            crossorigin="anonymous"></script>
+    </div>
+    </div>
+    </div>
+</body>
+
+</html>

templates/users_overview.html

@@ -1 +0,0 @@
-allah