Archos/masto-fe-archos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit 'dd72a8d28b4479afdc19ac73cac83609d85b5f9e' into glitch-soc/merge-upstream

Browse Source
This commit is contained in:
Claire
2023-09-02 12:02:23 +02:00
parent 3c7dbf3a16 dd72a8d28b
commit c8aa3d6d06
18 changed files with 274 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/chewy/accounts_index.rb
View File

@@ -21,19 +21,20 @@ class AccountsIndex < Chewy::Index
analyzer: {
natural: {
tokenizer: 'uax_url_email',
tokenizer: 'standard',
filter: %w(
english_possessive_stemmer
lowercase
asciifolding
cjk_width
elision
english_possessive_stemmer
english_stop
english_stemmer
),
},
verbatim: {
tokenizer: 'standard',
tokenizer: 'uax_url_email',
filter: %w(lowercase asciifolding cjk_width),
},
@@ -62,6 +63,6 @@ class AccountsIndex < Chewy::Index
field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
field(:text, type: 'text', analyzer: 'whitespace', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
field(:text, type: 'text', analyzer: 'verbatim', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
end
end

14
app/chewy/public_statuses_index.rb
View File

@@ -20,13 +20,19 @@ class PublicStatusesIndex < Chewy::Index
},
analyzer: {
content: {
verbatim: {
tokenizer: 'uax_url_email',
filter: %w(lowercase),
},
content: {
tokenizer: 'standard',
filter: %w(
english_possessive_stemmer
lowercase
asciifolding
cjk_width
elision
english_possessive_stemmer
english_stop
english_stemmer
),
@@ -40,9 +46,9 @@ class PublicStatusesIndex < Chewy::Index
.includes(:media_attachments, :preloadable_poll, :preview_cards)
root date_detection: false do
field(:id, type: 'keyword')
field(:id, type: 'long')
field(:account_id, type: 'long')
field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
field(:language, type: 'keyword')
field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
field(:created_at, type: 'date')

14
app/chewy/statuses_index.rb
View File

@@ -20,13 +20,19 @@ class StatusesIndex < Chewy::Index
},
analyzer: {
content: {
verbatim: {
tokenizer: 'uax_url_email',
filter: %w(lowercase),
},
content: {
tokenizer: 'standard',
filter: %w(
english_possessive_stemmer
lowercase
asciifolding
cjk_width
elision
english_possessive_stemmer
english_stop
english_stemmer
),
@@ -64,9 +70,9 @@ class StatusesIndex < Chewy::Index
end
root date_detection: false do
field(:id, type: 'keyword')
field(:id, type: 'long')
field(:account_id, type: 'long')
field(:text, type: 'text', analyzer: 'whitespace', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
field(:searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) })
field(:language, type: 'keyword')
field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })

47
app/javascript/mastodon/features/compose/components/search.jsx
View File

@@ -1,7 +1,7 @@
import PropTypes from 'prop-types';
import { PureComponent } from 'react';
import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
import { defineMessages, injectIntl, FormattedMessage, FormattedList } from 'react-intl';
import classNames from 'classnames';
@@ -45,6 +45,16 @@ class Search extends PureComponent {
options: [],
};
defaultOptions = [
{ label: <><mark>has:</mark> <FormattedList type='disjunction' value={['media', 'poll', 'embed']} /></>, action: e => { e.preventDefault(); this._insertText('has:') } },
{ label: <><mark>is:</mark> <FormattedList type='disjunction' value={['reply', 'sensitive']} /></>, action: e => { e.preventDefault(); this._insertText('is:') } },
{ label: <><mark>language:</mark> <FormattedMessage id='search_popout.language_code' defaultMessage='ISO language code' /></>, action: e => { e.preventDefault(); this._insertText('language:') } },
{ label: <><mark>from:</mark> <FormattedMessage id='search_popout.user' defaultMessage='user' /></>, action: e => { e.preventDefault(); this._insertText('from:') } },
{ label: <><mark>before:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('before:') } },
{ label: <><mark>during:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('during:') } },
{ label: <><mark>after:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('after:') } },
];
setRef = c => {
this.searchForm = c;
};
@@ -70,7 +80,7 @@ class Search extends PureComponent {
handleKeyDown = (e) => {
const { selectedOption } = this.state;
const options = this._getOptions();
const options = this._getOptions().concat(this.defaultOptions);
switch(e.key) {
case 'Escape':
@@ -100,11 +110,9 @@ class Search extends PureComponent {
if (selectedOption === -1) {
this._submit();
} else if (options.length > 0) {
options[selectedOption].action();
options[selectedOption].action(e);
}
this._unfocus();
break;
case 'Delete':
if (selectedOption > -1 && options.length > 0) {
@@ -147,6 +155,7 @@ class Search extends PureComponent {
router.history.push(`/tags/${query}`);
onClickSearchResult(query, 'hashtag');
this._unfocus();
};
handleAccountClick = () => {
@@ -157,6 +166,7 @@ class Search extends PureComponent {
router.history.push(`/@${query}`);
onClickSearchResult(query, 'account');
this._unfocus();
};
handleURLClick = () => {
@@ -164,6 +174,7 @@ class Search extends PureComponent {
const { value, onOpenURL } = this.props;
onOpenURL(value, router.history);
this._unfocus();
};
handleStatusSearch = () => {
@@ -182,6 +193,8 @@ class Search extends PureComponent {
} else if (search.get('type') === 'hashtag') {
router.history.push(`/tags/${search.get('q')}`);
}
this._unfocus();
};
handleForgetRecentSearchClick = search => {
@@ -194,6 +207,18 @@ class Search extends PureComponent {
document.querySelector('.ui').parentElement.focus();
}
_insertText (text) {
const { value, onChange } = this.props;
if (value === '') {
onChange(text);
} else if (value[value.length - 1] === ' ') {
onChange(`${value}${text}`);
} else {
onChange(`${value} ${text}`);
}
}
_submit (type) {
const { onSubmit, openInRoute } = this.props;
const { router } = this.context;
@@ -203,6 +228,8 @@ class Search extends PureComponent {
if (openInRoute) {
router.history.push('/search');
}
this._unfocus();
}
_getOptions () {
@@ -325,6 +352,16 @@ class Search extends PureComponent {
</div>
</>
)}
<h4><FormattedMessage id='search_popout.options' defaultMessage='Search options' /></h4>
<div className='search__popout__menu'>
{this.defaultOptions.map(({ key, label, action }, i) => (
<button key={key} onMouseDown={action} className={classNames('search__popout__menu__item', { selected: selectedOption === (options.length + i) })}>
{label}
</button>
))}
</div>
</div>
</div>
);

4
app/javascript/mastodon/locales/en.json
View File

@@ -586,8 +586,12 @@
"search.quick_action.open_url": "Open URL in Mastodon",
"search.quick_action.status_search": "Posts matching {x}",
"search.search_or_paste": "Search or paste URL",
"search_popout.language_code": "ISO language code",
"search_popout.options": "Search options",
"search_popout.quick_actions": "Quick actions",
"search_popout.recent": "Recent searches",
"search_popout.specific_date": "specific date",
"search_popout.user": "user",
"search_results.accounts": "Profiles",
"search_results.all": "All",
"search_results.hashtags": "Hashtags",

2
app/javascript/mastodon/test_setup.js
View File

@@ -1 +1 @@
import '@testing-library/jest-dom/extend-expect';
import '@testing-library/jest-dom';

6
app/javascript/styles/mastodon/components.scss
View File

@@ -4991,6 +4991,12 @@ a.status-card {
}
&__menu {
margin-bottom: 20px;
&:last-child {
margin-bottom: 0;
}
&__message {
color: $dark-text-color;
padding: 0 10px;

2
app/lib/search_query_parser.rb
View File

@@ -9,7 +9,7 @@ class SearchQueryParser < Parslet::Parser
rule(:prefix) { (term >> colon).as(:prefix) }
rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
rule(:clause) { (operator.maybe >> prefix.maybe >> (phrase | term | shortcode)).as(:clause) }
rule(:query) { (clause >> space.maybe).repeat.as(:query) }
root(:query)
end

24
app/lib/search_query_transformer.rb
View File

@@ -36,7 +36,11 @@ class SearchQueryTransformer < Parslet::Transform
def clause_to_filter(clause)
case clause
when PrefixClause
{ clause.type => { clause.filter => clause.term } }
if clause.negated?
{ bool: { must_not: { clause.type => { clause.filter => clause.term } } } }
else
{ clause.type => { clause.filter => clause.term } }
end
else
raise "Unexpected clause type: #{clause}"
end
@@ -81,7 +85,9 @@ class SearchQueryTransformer < Parslet::Transform
class PrefixClause
attr_reader :type, :filter, :operator, :term
def initialize(prefix, term)
def initialize(prefix, operator, term, options = {})
@negated = operator == '-'
@options = options
@operator = :filter
case prefix
@@ -100,23 +106,29 @@ class SearchQueryTransformer < Parslet::Transform
when 'before'
@filter = :created_at
@type = :range
@term = { lt: term }
@term = { lt: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
when 'after'
@filter = :created_at
@type = :range
@term = { gt: term }
@term = { gt: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
when 'during'
@filter = :created_at
@type = :range
@term = { gte: term, lte: term }
@term = { gte: term, lte: term, time_zone: @options[:current_account]&.user_time_zone || 'UTC' }
else
raise Mastodon::SyntaxError
end
end
def negated?
@negated
end
private
def account_id_from_term(term)
return @options[:current_account]&.id || -1 if term == 'me'
username, domain = term.gsub(/\A@/, '').split('@')
domain = nil if TagManager.instance.local_domain?(domain)
account = Account.find_remote(username, domain)
@@ -132,7 +144,7 @@ class SearchQueryTransformer < Parslet::Transform
operator = clause[:operator]&.to_s
if clause[:prefix]
PrefixClause.new(prefix, clause[:term].to_s)
PrefixClause.new(prefix, operator, clause[:term].to_s, current_account: current_account)
elsif clause[:term]
TermClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:shortcode]

2
app/models/concerns/account_statuses_search.rb
View File

@@ -32,7 +32,7 @@ module AccountStatusesSearch
return unless Chewy.enabled?
statuses.without_reblogs.where(visibility: :public).find_in_batches do |batch|
PublicStatusesIndex.import(query: batch)
PublicStatusesIndex.import(batch)
end
end

14
app/services/search_service.rb
View File

@@ -17,7 +17,7 @@ class SearchService < BaseService
results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
elsif @query.present?
results[:accounts] = perform_accounts_search! if account_searchable?
results[:statuses] = perform_statuses_search! if full_text_searchable?
results[:statuses] = perform_statuses_search! if status_searchable?
results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
end
end
@@ -79,18 +79,16 @@ class SearchService < BaseService
url_resource.class.name.downcase.pluralize.to_sym
end
def full_text_searchable?
return false unless Chewy.enabled?
statuses_search? && !@account.nil? && !(@query.include?('@') && !@query.include?(' '))
def status_searchable?
Chewy.enabled? && status_search? && @account.present?
end
def account_searchable?
account_search? && !(@query.include?('@') && @query.include?(' '))
account_search?
end
def hashtag_searchable?
hashtag_search? && !@query.include?('@')
hashtag_search?
end
def account_search?
@@ -101,7 +99,7 @@ class SearchService < BaseService
@options[:type].blank? || @options[:type] == 'hashtags'
end
def statuses_search?
def status_search?
@options[:type].blank? || @options[:type] == 'statuses'
end
end

2
app/services/statuses_search_service.rb
View File

@@ -59,6 +59,6 @@ class StatusesSearchService < BaseService
end
def parsed_query
SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query), current_account: @account)
end
end

11
app/workers/add_to_public_statuses_index_worker.rb
View File

@@ -2,15 +2,20 @@
class AddToPublicStatusesIndexWorker
include Sidekiq::Worker
include DatabaseHelper
sidekiq_options queue: 'pull'
def perform(account_id)
account = Account.find(account_id)
with_primary do
@account = Account.find(account_id)
end
return unless account.indexable?
return unless @account.indexable?
account.add_to_public_statuses_index!
with_read_replica do
@account.add_to_public_statuses_index!
end
rescue ActiveRecord::RecordNotFound
true
end

6
app/workers/scheduler/indexing_scheduler.rb
View File

@@ -3,6 +3,7 @@
class Scheduler::IndexingScheduler
include Sidekiq::Worker
include Redisable
include DatabaseHelper
sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
@@ -15,7 +16,10 @@ class Scheduler::IndexingScheduler
indexes.each do |type|
with_redis do |redis|
redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
type.import!(ids)
with_read_replica do
type.import!(ids)
end
redis.srem("chewy:queue:#{type.name}", ids)
end
end