Skip to content

Commit cb5797b

Browse files
aaxelbaaxelb
committed
better match elasticsearch_metrics changes
1 parent 08258b7 commit cb5797b

18 files changed

Lines changed: 117 additions & 59 deletions

‎addons/base/views.py‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -693,13 +693,15 @@ def osfstoragefile_viewed_update_metrics(self, auth, fileversion, file_node):
693693
path=file_node.path,
694694
)
695695
OsfCountedUsageRecord.record(
696-
count=1,
697-
preprint_id=resource._id,
698-
user_id=getattr(auth.user, '_id', None),
699-
provider_id=resource.provider._id,
700-
database_iri=resource.get_semantic_iri(),
701-
version=fileversion.identifier,
702-
path=file_node.path,
696+
user_id=getattr(user, '_id', None),
697+
item_osfid=resource._id,
698+
action_labels=[
699+
OsfCountedUsageRecord.ActionLabel.VIEW.value,
700+
OsfCountedUsageRecord.ActionLabel.WEB.value,
701+
],
702+
# HACK: we don't have the user request, so fabricate a one-off session id
703+
# (this means no double-click filtering and inflated "unique" view counts)
704+
client_session_id=str(uuid.uuid4()),
703705
)
704706
except es_exceptions.ConnectionError:
705707
log_exception()
@@ -729,13 +731,14 @@ def osfstoragefile_downloaded_update_metrics(self, auth, fileversion, file_node)
729731
path=file_node.path,
730732
)
731733
OsfCountedUsageRecord.record(
732-
count=1,
733-
preprint_id=resource._id,
734-
user_id=getattr(auth.user, '_id', None),
735-
provider_id=resource.provider._id,
736-
database_iri=resource.get_semantic_iri(),
737-
version=fileversion.identifier,
738-
path=file_node.path,
734+
user_id=getattr(user, '_id', None),
735+
item_osfid=resource._id,
736+
action_labels=[
737+
OsfCountedUsageRecord.ActionLabel.DOWNLOAD.value,
738+
],
739+
# HACK: we don't have the user request, so fabricate a one-off session id
740+
# (this means no double-click filtering and inflated "unique" download counts)
741+
client_session_id=str(uuid.uuid4()),
739742
)
740743
except es_exceptions.ConnectionError:
741744
log_exception()

‎api/metrics/serializers.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,8 @@ def create(self, validated_data):
7373
pageview_info = PageviewInfo(**pageview_info_data)
7474
pageview_info_es8 = PageviewInfoEs8(**pageview_info_data)
7575
OsfCountedUsageRecord.record(
76-
platform_iri=website_settings.DOMAIN,
77-
provider_id=validated_data.get('provider_id'),
7876
item_osfid=validated_data.get('item_guid'),
79-
sessionhour_id=validated_data['session_id'],
80-
user_is_authenticated=validated_data['user_is_authenticated'],
77+
client_session_id=validated_data['session_id'],
8178
action_labels=validated_data.get('action_labels'),
8279
pageview_info=pageview_info_es8,
8380
)

‎osf/metrics/es8_metrics.py‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,28 @@ class OsfCountedUsageRecord(djelme.CountedUsageRecord):
111111
class Meta:
112112
timeseries_index_timedepth = MONTHLY
113113

114+
class ActionLabel(enum.Enum):
115+
SEARCH = 'search' # counter:Search
116+
VIEW = 'view' # counter:Investigation
117+
DOWNLOAD = 'download' # counter:Request
118+
WEB = 'web' # counter:Regular (aka "pageview")
119+
API = 'api' # counter:TDM (aka "non-web api usage")
120+
121+
@classmethod
122+
def record(cls, **kwargs):
123+
# autofill `user_is_authenticated` before `user_id` discarded (couldn't in `clean`)
124+
if 'user_is_authenticated' not in kwargs:
125+
kwargs['user_is_authenticated'] = bool(kwargs.get('user_id'))
126+
return super().record(**kwargs)
127+
114128
@functools.cached_property
115129
def _osfid_referent(self):
116130
# for use by autofill methods, if needed
117131
return osfdb.Guid.load(self.item_osfid)
118132

119133
def clean(self):
120134
super().clean()
135+
self._autofill_platform_iri()
121136
self._autofill_item_iri_and_osfid()
122137
self._autofill_item_public()
123138
self._autofill_item_type()
@@ -126,6 +141,10 @@ def clean(self):
126141
self._autofill_pageview()
127142
self._autofill_database_iri()
128143

144+
def _autofill_platform_iri(self):
145+
if self.platform_iri is None:
146+
self.platform_iri = website_settings.DOMAIN
147+
129148
def _autofill_item_iri_and_osfid(self):
130149
if self.item_osfid and not self.item_iri:
131150
self.item_iri = osf_iri(self.item_osfid)
@@ -227,14 +246,6 @@ def _get_unique_together_values(self):
227246
)
228247

229248

230-
class ActionLabel(enum.Enum):
231-
SEARCH = 'search' # counter:Search
232-
VIEW = 'view' # counter:Investigation
233-
DOWNLOAD = 'download' # counter:Request
234-
WEB = 'web' # counter:Regular (aka "pageview")
235-
API = 'api' # counter:TDM (aka "non-web api usage")
236-
237-
238249
class RegistriesModerationMetricsEs8(djelme.EventRecord):
239250
registration_id: str
240251
provider_id: str

‎osf/metrics/reporters/download_count.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
from osf.models import PageCounter
22
from osf.metrics.reports import DownloadCountReport
3-
from ._base import DailyReporter
43
from osf.metrics.es8_metrics import DownloadCountReportEs8
4+
from osf.metrics.utils import cycle_coverage_date
5+
from ._base import DailyReporter
56

67

78
class DownloadCountReporter(DailyReporter):
89
def report(self, date):
910
download_count = int(PageCounter.get_all_downloads_on_date(date) or 0)
1011
reports = []
1112
report_es8 = DownloadCountReportEs8(
12-
cycle_coverage=f"{date:%Y.%m.%d}",
13+
cycle_coverage=cycle_coverage_date(date),
1314
daily_file_downloads=download_count,
1415
)
1516
reports.append(report_es8)

‎osf/metrics/reporters/institution_summary.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,14 @@
99
RegistrationRunningTotals,
1010
)
1111
from osf.models import Institution
12-
from ._base import DailyReporter
1312
from osf.metrics.es8_metrics import (
1413
InstitutionSummaryReportEs8,
1514
RunningTotal as RunningTotalEs8,
1615
NodeRunningTotals as NodeRunningTotalsEs8,
1716
RegistrationRunningTotals as RegistrationRunningTotalsEs8
1817
)
18+
from osf.metrics.utils import cycle_coverage_date
19+
from ._base import DailyReporter
1920

2021
logger = logging.getLogger(__name__)
2122
logging.basicConfig(level=logging.INFO)
@@ -45,7 +46,7 @@ def report(self, date):
4546
type='osf.registration',
4647
)
4748
report_es8 = InstitutionSummaryReportEs8(
48-
cycle_coverage=f"{date:%Y.%m.%d}",
49+
cycle_coverage=cycle_coverage_date(date),
4950
institution_id=institution._id,
5051
institution_name=institution.name,
5152
users=RunningTotalEs8(

‎osf/metrics/reporters/institution_summary_monthly.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@
55
from osf.models.spam import SpamStatus
66
from addons.osfstorage.models import OsfStorageFile
77
from osf.metrics.reports import InstitutionMonthlySummaryReport
8-
from ._base import MonthlyReporter
98
from osf.metrics.es8_metrics import InstitutionMonthlySummaryReportEs8
10-
9+
from osf.metrics.utils import cycle_coverage_yearmonth
10+
from ._base import MonthlyReporter
1111

1212
class InstitutionalSummaryMonthlyReporter(MonthlyReporter):
1313
"""Generate an InstitutionMonthlySummaryReport for each institution."""
@@ -35,7 +35,7 @@ def generate_report(self, institution):
3535
preprint_queryset = self.get_published_preprints(institution, self.yearmonth)
3636
reports = []
3737
report_es8 = InstitutionMonthlySummaryReportEs8(
38-
cycle_coverage=f"{self.yearmonth.year}.{self.yearmonth.month}",
38+
cycle_coverage=cycle_coverage_yearmonth(self.yearmonth),
3939
institution_id=institution._id,
4040
user_count=institution.get_institution_users().count(),
4141
private_project_count=self._get_count(node_queryset, 'osf.node', is_public=False),

‎osf/metrics/reporters/institutional_users.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88
from osf.models.spam import SpamStatus
99
from addons.osfstorage.models import OsfStorageFile
1010
from osf.metrics.reports import InstitutionalUserReport
11-
from osf.metrics.utils import YearMonth
12-
from ._base import MonthlyReporter
11+
from osf.metrics.utils import YearMonth, cycle_coverage_yearmonth
1312
from osf.metrics.es8_metrics import InstitutionalUserReportEs8
13+
from ._base import MonthlyReporter
1414

1515

1616
class InstitutionalUsersReporter(MonthlyReporter):
@@ -56,7 +56,7 @@ def __post_init__(self):
5656
_affiliation = self.user.get_institution_affiliation(self.institution._id)
5757
self.reports = []
5858
report_es8 = InstitutionalUserReportEs8(
59-
cycle_coverage=f"{self.yearmonth.year}.{self.yearmonth.month}",
59+
cycle_coverage=cycle_coverage_yearmonth(self.yearmonth),
6060
institution_id=self.institution._id,
6161
user_id=self.user._id,
6262
user_name=self.user.fullname,

‎osf/metrics/reporters/new_user_domain.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33

44
from osf.models import OSFUser
55
from osf.metrics.reports import NewUserDomainReport
6-
from ._base import DailyReporter
76
from osf.metrics.es8_metrics import NewUserDomainReportEs8
7+
from osf.metrics.utils import cycle_coverage_date
8+
from ._base import DailyReporter
89

910
logger = logging.getLogger(__name__)
1011
logging.basicConfig(level=logging.INFO)
@@ -24,7 +25,7 @@ def report(self, date):
2425
reports = []
2526
for domain_name, count in domain_names.items():
2627
report_es8 = NewUserDomainReportEs8(
27-
cycle_coverage=f"{date:%Y.%m.%d}",
28+
cycle_coverage=cycle_coverage_date(date),
2829
domain_name=domain_name,
2930
new_user_count=count,
3031
)

‎osf/metrics/reporters/node_count.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@
77
NodeRunningTotals,
88
RegistrationRunningTotals,
99
)
10-
from ._base import DailyReporter
1110
from osf.metrics.es8_metrics import (
1211
NodeSummaryReportEs8,
1312
NodeRunningTotals as NodeRunningTotalsEs8,
1413
RegistrationRunningTotals as RegistrationRunningTotalsEs8
1514
)
15+
from osf.metrics.utils import cycle_coverage_date
16+
from ._base import DailyReporter
1617

1718
logger = logging.getLogger(__name__)
1819
logging.basicConfig(level=logging.INFO)
@@ -41,7 +42,7 @@ def report(self, date):
4142
exclude_spam = ~Q(spam_status__in=[SpamStatus.SPAM, SpamStatus.FLAGGED])
4243
reports = []
4344
report_es8 = NodeSummaryReportEs8(
44-
cycle_coverage=f"{date:%Y.%m.%d}",
45+
cycle_coverage=cycle_coverage_date(date),
4546
# Nodes - the number of projects and components
4647
nodes=NodeRunningTotalsEs8(
4748
total=node_qs.count(),

‎osf/metrics/reporters/osfstorage_file_count.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44

55
from osf.metrics.reports import OsfstorageFileCountReport, FileRunningTotals
66
from osf.models import AbstractNode, Preprint
7-
from ._base import DailyReporter
87
from osf.metrics.es8_metrics import (
98
OsfstorageFileCountReportEs8,
109
FileRunningTotals as FileRunningTotalsEs8
1110
)
11+
from osf.metrics.utils import cycle_coverage_date
12+
from ._base import DailyReporter
1213

1314
logger = logging.getLogger(__name__)
1415
logging.basicConfig(level=logging.INFO)
@@ -38,7 +39,7 @@ def report(self, date):
3839
reports = []
3940

4041
report_es8 = OsfstorageFileCountReportEs8(
41-
cycle_coverage=f"{date:%Y.%m.%d}",
42+
cycle_coverage=cycle_coverage_date(date),
4243
files=FileRunningTotalsEs8(
4344
total=file_qs.count(),
4445
public=file_qs.filter(public_query).count(),

0 commit comments

Comments
 (0)
⚡