2ch statistic page has been fixed and improved.

This commit is contained in:
AB
2018-10-31 01:42:02 +03:00
parent a92daa87ac
commit e6be431049
6 changed files with 272 additions and 231 deletions

View File

@ -1,7 +1,9 @@
import datetime
import logging
import os
import requests
import time
from flask import Flask, request, send_from_directory
from flask import render_template
@ -35,7 +37,7 @@ def serve_static(path):
return send_from_directory('static', path)
@app.route('/')
@app.route('/', methods=['GET', 'POST'])
def index():
order = request.args.get('order', default='id', type=str)
sorting = request.args.get('sorting', default='ASC', type=str)
@ -136,6 +138,7 @@ def stat():
sorting=sorting,
posts=posts,
threads=sorted_threads,
now=datetime.datetime.now().strftime('%s'),
)
@ -143,5 +146,46 @@ def main():
app.run(host=flask_host, port=flask_port)
@app.template_filter('datetimeformat')
def datetimeformat(value, format='%H:%M / %d-%m-%Y'):
return datetime.datetime.fromtimestamp(value).strftime(format)
@app.template_filter('readable_delta')
def readable_delta(from_seconds, until_seconds=None):
# Returns a nice readable delta.
def plur(it):
try:
size = len(it)
except TypeError:
size = int(it)
return '' if size == 1 else 's'
if not until_seconds:
until_seconds = time.time()
seconds = int(until_seconds) - int(from_seconds)
delta = datetime.timedelta(seconds=seconds)
delta_minutes = delta.seconds // 60
delta_hours = delta_minutes // 60
if delta.days:
return '%d day%s, %d hour%s' % (
delta.days,
plur(delta.days),
(delta_hours),
plur(delta_hours))
elif delta_hours:
return '%d hour%s, %d minute%s' % (
delta_hours,
plur(delta_hours),
(delta_minutes - delta_hours * 60),
plur(delta_minutes))
elif delta_minutes:
return '%d minute%s' % (delta_minutes, plur(delta_minutes))
else:
return '%d second%s' % (delta.seconds, plur(delta.seconds))
if __name__ == '__main__':
main()

View File

@ -1,31 +1,30 @@
<!DOCTYPE html>
<html lang="en">
<head>
{% block head %}
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO"
crossorigin="anonymous">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO"
crossorigin="anonymous">
<link rel="stylesheet" type="text/css" href="../static/css/style.css">
<title>libOpenAnal appliance</title>
<link rel="stylesheet" type="text/css" href="../static/css/style.css">
<title>libOpenAnal appliance</title>
{% endblock %}
</head>
<body>
<div class="container" id="content">{% block content %}
<h1>Hexor's conf_bot data extractor tool</h1>{% endblock %}</div>
{% block scripts %}
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
{% endblock %}
<div class="container" id="content">{% block content %}
<h1>Hexor's conf_bot data extractor tool</h1>{% endblock %}</div>
{% block scripts %}
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
crossorigin="anonymous"></script>
{% endblock %}
</body>
</html>

View File

@ -1,37 +1,36 @@
{% extends "base.html" %}
{% block head %}
{{ super() }}
{{ super() }}
{% endblock %}
{% block content %}
{{ super() }}
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link" href="/">Users</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="/conf">Conferences</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/stat">2ch.hk Stats</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#">Disabled</a>
</li>
</ul>
<br>
<div class="row">
<div class="col-sm-12">
<h4>Totals</h4>
<b>Users: </b> {{ totals.users[0] }}<br>
<b>Unique words: </b> {{ totals.words[0] }}<br>
<b>Words said: </b> {{ totals.relations[0] }}<br>
<b>Chats fetched: </b> {{ totals.confs[0] }}<br>
</div>
{{ super() }}
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link" href="/">Users</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="/conf">Conferences</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/stat">2ch.hk Stats</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#">Disabled</a>
</li>
</ul>
<br>
<div class="row">
<div class="col-sm-12">
<h4>Summary</h4>
<b>Users: </b> {{ totals.users[0] }}<br>
<b>Unique words: </b> {{ totals.words[0] }}<br>
<b>Words said: </b> {{ totals.relations[0] }}<br>
<b>Chats fetched: </b> {{ totals.confs[0] }}<br>
</div>
</div>
<table class="table table-hover table-sm">
<thead>
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col">Title</th>
@ -40,21 +39,21 @@
<th scope="col">Members</th>
<th scope="col">Words said</th>
</tr>
</thead>
<tbody>
</thead>
<tbody>
{% for conf in confs %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td>{{ conf.0 }}</td>
<td>{{ conf.1 }}</td>
<td>{{ conf.2 }}</td>
<td>{{ conf.4 }}</td>
<td>{{ conf.3 }}</td>
</tr>
<tr>
<th scope="row">{{ loop.index }}</th>
<td>{{ conf.0 }}</td>
<td>{{ conf.1 }}</td>
<td>{{ conf.2 }}</td>
<td>{{ conf.4 }}</td>
<td>{{ conf.3 }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</tbody>
</table>
{% endblock %}
{% block scripts %}
{{ super() }}
{{ super() }}
{% endblock %}

View File

@ -1,78 +1,72 @@
{% extends "base.html" %}
{% block head %}
{{ super() }}
{{ super() }}
{% endblock %}
{% block content %}
{{ super() }}
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link active" href="/">Users</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/conf">Conferences</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/stat">2ch.hk Stats</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#">Disabled</a>
</li>
</ul>
<br>
<div class="row">
<div class="col-sm-12">
<h4>Totals</h4>
<b>Users: </b> {{ totals.users[0] }}<br>
<b>Unique words: </b> {{ totals.words[0] }}<br>
<b>Words said: </b> {{ totals.relations[0] }}<br>
<b>Chats fetched: </b> {{ totals.confs[0] }}<br>
</div>
{{ super() }}
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link active" href="/">Users</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/conf">Conferences</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/stat">2ch.hk Stats</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#">Disabled</a>
</li>
</ul>
<br>
<div class="row">
<div class="col-sm-12">
<h4>Summary</h4>
<b>Users: </b> {{ totals.users[0] }}<br>
<b>Unique words: </b> {{ totals.words[0] }}<br>
<b>Words said: </b> {{ totals.relations[0] }}<br>
<b>Chats fetched: </b> {{ totals.confs[0] }}<br>
</div>
<br>
<table class="table table-hover table-sm">
<thead>
</div>
<br>
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col"><a
href="./?order=first_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">First
name</a></th>
<th scope="col"><a
href="./?order=last_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last
name</a></th>
<th scope="col"><a
href="./?order=username&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Username</a>
<th scope="col"><a href="./?order=first_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">First
name</a></th>
<th scope="col"><a href="./?order=last_name&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last
name</a></th>
<th scope="col"><a href="./?order=username&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Username</a>
</th>
<th scope="col"><a href="./?order=id&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">User
ID</a></th>
<th scope="col"><a
href="./?order=firstly_seen&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Firstly
seen</a></th>
<th scope="col"><a
href="./?order=last_activity&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last
activity</a></th>
<th scope="col"><a
href="./?order=count&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Word
count</a></th>
ID</a></th>
<th scope="col"><a href="./?order=firstly_seen&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Firstly
seen</a></th>
<th scope="col"><a href="./?order=last_activity&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Last
activity</a></th>
<th scope="col"><a href="./?order=count&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Word
count</a></th>
</tr>
</thead>
<tbody>
</thead>
<tbody>
{% for user in users %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td><a href="/overview/user/{{ user.0 }}">{{ user.2 }}</a></td>
<td>{% if user.3 != '_null' %}{{ user.3 }}{% else %}
<span class="badge badge-warning">N/D</span>{% endif %}</td>
<td>{% if user.1 != '_null' %}{{ user.1 }}{% else %}
<span class="badge badge-warning">N/D</span>{% endif %}</td>
<td><a class="badge badge-dark " href="/overview/user/{{ user.0 }}">{{ user.0 }}</a></td>
<td>{{ user.4 }}</td>
<td>{{ user.5 }}</td>
<td>{{ user.6 }}</td>
</tr>
<tr>
<th scope="row">{{ loop.index }}</th>
<td><a href="/overview/user/{{ user.0 }}">{{ user.2 }}</a></td>
<td>{% if user.3 != '_null' %}{{ user.3 }}{% else %}
<span class="badge badge-warning">N/D</span>{% endif %}</td>
<td>{% if user.1 != '_null' %}{{ user.1 }}{% else %}
<span class="badge badge-warning">N/D</span>{% endif %}</td>
<td><a class="badge badge-dark " href="/overview/user/{{ user.0 }}">{{ user.0 }}</a></td>
<td>{{ user.4 }}</td>
<td>{{ user.5 }}</td>
<td>{{ user.6 }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</tbody>
</table>
{% endblock %}
{% block scripts %}
{{ super() }}
{{ super() }}
{% endblock %}

View File

@ -1,79 +1,84 @@
{% extends "base.html" %}
{% block head %}
{{ super() }}
{{ super() }}
{% endblock %}
{% block content %}
{{ super() }}
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link" href="/">Users</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/conf">Conferences</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="/stat">2ch.hk Stats</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#">Disabled</a>
</li>
</ul>
<br>
<form action="/" method="get">
<div class="input-group mb-3">
<input type="text" class="form-control" placeholder="Board" aria-label="Board"
aria-describedby="button-stat" name="board">
<div class="input-group-append">
<button class="btn btn-primary" type="submit" id="button-stat">Get stat</button>
</div>
</div>
</form>
{{ super() }}
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link" href="/">Users</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/conf">Conferences</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="/stat">2ch.hk Stats</a>
</li>
<li class="nav-item">
<a class="nav-link disabled" href="#">Disabled</a>
</li>
</ul>
<br>
<div class="row">
<div class="col-sm-12">
<h4>Summary</h4>
<b>Board: </b> {{ board }}<br>
<b>Threads: </b> {{ threads|length }}<br>
<b>Total posts: </b> {{ posts }}<br>
</div>
</div>
<hr>
<div class="container">
<div class="row">
<div class="col-sm-12">
<h4>Info</h4>
<b>Board: </b> {{ board }}<br>
<b>Threads: </b> {{ threads|length }}<br>
<b>Total posts: </b> {{ posts }}<br>
<div class="col-md-4"></div>
<div class="col-md-4 offset-md-4">
<form action="/stat" method="get">
<div class="input-group mb-3">
<input type="text" class="form-control" placeholder="Board" aria-label="Board" aria-describedby="button-stat"
name="board">
<div class="input-group-append">
<button class="btn btn-primary" type="submit" id="button-stat">Get stat</button>
</div>
</div>
</form>
</div>
</div>
<br>
<table class="table table-hover table-sm">
<thead>
</div>
<br>
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col">Subject</th>
<th scope="col"><a
href="./stat?board={{ board }}&order=posts&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Posts</a>
<th scope="col"><a href="./stat?board={{ board }}&order=posts&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Posts</a>
</th>
<th scope="col"><a
href="./stat?board={{ board }}&order=views&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Views</a>
<th scope="col"><a href="./stat?board={{ board }}&order=views&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Views</a>
</th>
<th scope="col"><a
href="./stat?board={{ board }}&order=score&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Score</a>
<th scope="col"><a href="./stat?board={{ board }}&order=score&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Score</a>
</th>
<th scope="col"><a
href="./stat?board={{ board }}&order=timestamp&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Created</a>
</th>
<th scope="col"><a
href="./stat?board={{ board }}&order=lasthit&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Lasthit</a>
<th scope="col"><a href="./stat?board={{ board }}&order=lasthit&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Lasthit</a>
<th scope="col"><a href="./stat?board={{ board }}&order=timestamp&sorting={%- if sorting == 'ASC' -%}DESC{%- else -%}ASC{%- endif -%}">Age</a>
</th>
</tr>
</thead>
<tbody>
</thead>
<tbody>
{% for thread in threads %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td><a href="https://2ch.hk/b/res/{{ thread['num'] }}.html">{{ thread['subject'] }}</a></td>
<td>{{ thread['posts_count'] }}</td>
<td>{{ thread['views'] }}</td>
<td>{{ "%.2f"|format(thread['score']|float) }}</td>
<td>{{ thread['timestamp'] | time }}</td>
<td>{{ thread['lasthit'] | time }}</td>
</tr>
<tr>
<th scope="row">{{ loop.index }}</th>
<td><a href="https://2ch.hk/{{board}}/res/{{ thread['num'] }}.html">{{ thread['subject'] }}</a></td>
<td>{{ thread['posts_count'] }}</td>
<td>{{ thread['views'] }}</td>
<td>{{ "%.2f"|format(thread['score']|float) }}</td>
<td data-toggle="tooltip" data-placement="right" title="Last hit: {{ thread['lasthit']|time }}">{{thread['lasthit']|readable_delta(now)}}
ago
</td>
<td data-toggle="tooltip" data-placement="right" title="Created: {{ thread['timestamp']|time }}">{{thread['timestamp']|readable_delta(thread['lasthit'])}}</td>
<td></td>
</tr>
{% endfor %}
</tbody>
</table>
</tbody>
</table>
{% endblock %}
{% block scripts %}
{{ super() }}
{{ super() }}
{% endblock %}

View File

@ -1,6 +1,6 @@
{% extends "base.html" %}
{% block head %}
{{ super() }}
{{ super() }}
{% endblock %}
{% block content %}
{{ super() }}
@ -58,59 +58,59 @@
<h5 class="card-title">Top</h5>
<hr>
<p class="card-text">
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col">Word</th>
<th scope="col">Said</th>
</tr>
</thead>
<tbody>
{% for word in user_info.top %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td>{{ word[0] }}</td>
{% if not loop.last %}
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col">Word</th>
<th scope="col">Said</th>
</tr>
</thead>
<tbody>
{% for word in user_info.top %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td>{{ word[0] }}</td>
{% if not loop.last %}
{% if (word[1]/loop.nextitem[1]) > 2 %}
<td><span class="badge badge-danger" data-toggle="tooltip" data-placement="right"
title="Must have been abused">{{ word[1] }} </span></td>
<td><span class="badge badge-danger" data-toggle="tooltip" data-placement="right" title="Must have been abused">{{
word[1] }} </span></td>
{% else %}
<td><span class="badge badge-secondary">{{ word[1] }} </span></td>
{% endif %}
{% else %}
<td><span class="badge badge-secondary">{{ word[1] }} </span></td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% else %}
<td><span class="badge badge-secondary">{{ word[1] }} </span></td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="card">
<div class="card-body">
<h5 class="card-title">Chats</h5>
<p class="card-text">
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col">Chat</th>
<th scope="col">Words</th>
<th scope="col">Messages</th>
</tr>
</thead>
<tbody>
{% for chat in user_info.chats %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td>{{ chat[0] }}</td>
<td><span class="badge badge-secondary">{{ chat[1] }} </span></td>
<td><span class="badge badge-secondary">{{ chat[3] }} </span></td>
</tr>
{% endfor %}
</tbody>
</table>
<table class="table table-hover table-sm">
<thead>
<tr>
<th scope="col">#</th>
<th scope="col">Chat</th>
<th scope="col">Words</th>
<th scope="col">Messages</th>
</tr>
</thead>
<tbody>
{% for chat in user_info.chats %}
<tr>
<th scope="row">{{ loop.index }}</th>
<td>{{ chat[0] }}</td>
<td><span class="badge badge-secondary">{{ chat[1] }} </span></td>
<td><span class="badge badge-secondary">{{ chat[3] }} </span></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>