Commit 7374661e authored by cmrd Senya's avatar cmrd Senya

Update the user data export archive format.

This commit introduces changes to the user data export archive format.
This extends data set which is included in the archive. This data can be
then imported to other pods when this feature is implemented.

Also the commit adds the archive format json schema. ATM it is used in
automatic tests only, but in future it will also be used to validate
incoming archives.
parent 1b1db3bb
......@@ -13,6 +13,7 @@ gem "unicorn-worker-killer", "0.4.4"
# Federation
gem "diaspora_federation-json_schema", "0.2.1"
gem "diaspora_federation-rails", "0.2.1"
# API and JSON
......@@ -277,6 +278,8 @@ group :test do
gem "fixture_builder", "0.5.0"
gem "fuubar", "2.2.0"
gem "json-schema-rspec", "0.0.4"
gem "rspec-json_expectations", "~> 2.1"
gem "test_after_commit", "1.1.0"
# Cucumber (integration tests)
......
......@@ -168,6 +168,7 @@ GEM
nokogiri (~> 1.6, >= 1.6.8)
typhoeus (~> 1.0)
valid (~> 1.0)
diaspora_federation-json_schema (0.2.1)
diaspora_federation-rails (0.2.1)
actionpack (>= 4.2, < 6)
diaspora_federation (= 0.2.1)
......@@ -334,6 +335,9 @@ GEM
url_safe_base64
json-schema (2.8.0)
addressable (>= 2.4)
json-schema-rspec (0.0.4)
json-schema (~> 2.5)
rspec
jsonpath (0.8.5)
multi_json
jwt (1.5.6)
......@@ -582,6 +586,7 @@ GEM
rspec-expectations (3.6.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.6.0)
rspec-json_expectations (2.1.0)
rspec-mocks (3.6.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.6.0)
......@@ -772,6 +777,7 @@ DEPENDENCIES
devise (= 4.3.0)
devise_lastseenable (= 0.0.6)
diaspora-prosody-config (= 0.0.7)
diaspora_federation-json_schema (= 0.2.1)
diaspora_federation-rails (= 0.2.1)
diaspora_federation-test (= 0.2.1)
entypo-rails (= 3.0.0)
......@@ -800,6 +806,7 @@ DEPENDENCIES
js_image_paths (= 0.1.0)
json (= 2.1.0)
json-schema (= 2.8.0)
json-schema-rspec (= 0.0.4)
leaflet-rails (= 1.1.0)
logging-rails (= 0.6.0)
markerb (= 1.1.0)
......@@ -858,6 +865,7 @@ DEPENDENCIES
rb-inotify (= 0.9.10)
redcarpet (= 3.4.0)
responders (= 2.4.0)
rspec-json_expectations (~> 2.1)
rspec-rails (= 3.6.0)
rubocop (= 0.49.1)
ruby-oembed (= 0.12.0)
......
......@@ -37,7 +37,9 @@ class Person < ActiveRecord::Base
has_many :posts, :foreign_key => :author_id, :dependent => :destroy # This person's own posts
has_many :photos, :foreign_key => :author_id, :dependent => :destroy # This person's own photos
has_many :comments, :foreign_key => :author_id, :dependent => :destroy # This person's own comments
has_many :likes, foreign_key: :author_id, dependent: :destroy # This person's own likes
has_many :participations, :foreign_key => :author_id, :dependent => :destroy
has_many :poll_participations, foreign_key: :author_id, dependent: :destroy
has_many :conversation_visibilities
has_many :roles
......
module Export
class CommentSerializer < ActiveModel::Serializer
attributes :guid,
:text,
:post_guid
def post_guid
object.post.guid
end
end
end
......@@ -2,11 +2,33 @@ module Export
class ContactSerializer < ActiveModel::Serializer
attributes :sharing,
:receiving,
:following,
:followed,
:person_guid,
:person_name,
:person_first_name,
:person_diaspora_handle
:account_id,
:public_key
has_many :aspects, each_serializer: Export::AspectSerializer
has_many :contact_groups_membership
def following
object.sharing
end
def followed
object.receiving
end
def account_id
object.person_diaspora_handle
end
def contact_groups_membership
object.aspects.map(&:name)
end
def public_key
object.person.serialized_public_key
end
end
end
module Export
class OthersDataSerializer < ActiveModel::Serializer
# Relayables of other people in the archive: comments, likes, participations, poll participations where author is
# the archive owner
has_many :relayables, each_serializer: FederationEntitySerializer
# Parent posts of user's own relayables. We have to save metadata to use
# it in case when posts temporary unavailable on the target pod.
has_many :posts, each_serializer: FederationEntitySerializer
# Authors of posts where we participated and authors are not in contacts
has_many :non_contact_authors, each_serializer: PersonMetadataSerializer
private
def relayables
%i[comments likes poll_participations].map {|relayable|
others_relayables.send(relayable)
}.sum
end
def others_relayables
@others_relayables ||= Diaspora::Exporter::OthersRelayables.new(object.person_id)
end
def posts
@posts ||= Diaspora::Exporter::PostsWithActivity.new(object).query
end
def non_contact_authors
Diaspora::Exporter::NonContactAuthors.new(posts, object).query
end
end
end
module Export
# This is a serializer for the user's own posts
class OwnPostSerializer < FederationEntitySerializer
# Only for public posts.
# Includes URIs of pods which must be notified on the post updates.
# Must always include local pod URI since we will want all the updates on the post if user migrates.
has_many :subscribed_pods_uris
# Only for private posts.
# Includes diaspora* IDs of people who must be notified on post updates.
has_many :subscribed_users_ids
# Normally accepts Post as an object.
def initialize(*)
super
self.except = [excluded_subscription_key]
end
private
def subscribed_pods_uris
object.subscribed_pods_uris.push(AppConfig.pod_uri.to_s)
end
def subscribed_users_ids
object.subscribers.map(&:diaspora_handle)
end
def excluded_subscription_key
entity.public ? :subscribed_users_ids : :subscribed_pods_uris
end
end
end
module Export
# This is a serializer for the user's own relayables. We remove signature from the own relayables since it isn't
# useful and takes space.
class OwnRelayablesSerializer < FederationEntitySerializer
private
def modify_serializable_object(hash)
super.tap {|hash|
hash[:entity_data].delete(:author_signature)
}
end
end
end
module Export
class PersonMetadataSerializer < ActiveModel::Serializer
attributes :guid,
:account_id,
:public_key
private
def account_id
object.diaspora_handle
end
def public_key
object.serialized_public_key
end
end
end
module Export
class PostSerializer < ActiveModel::Serializer
attributes :guid,
:text,
:public,
:diaspora_handle,
:type,
:likes_count,
:comments_count,
:reshares_count,
:created_at
end
end
module Export
class ProfileSerializer < ActiveModel::Serializer
attributes :first_name,
:last_name,
:gender,
:bio,
:birthday,
:location,
:image_url,
:diaspora_handle,
:searchable,
:nsfw
end
end
module Export
class UserSerializer < ActiveModel::Serializer
attributes :name,
attributes :username,
:email,
:language,
:username,
:serialized_private_key,
:private_key,
:disable_mail,
:show_community_spotlight_in_stream,
:auto_follow_back,
:auto_follow_back_aspect,
:strip_exif
has_one :profile, serializer: Export::ProfileSerializer
has_many :aspects, each_serializer: Export::AspectSerializer
has_one :profile, serializer: FederationEntitySerializer
has_many :contact_groups, each_serializer: Export::AspectSerializer
has_many :contacts, each_serializer: Export::ContactSerializer
has_many :posts, each_serializer: Export::PostSerializer
has_many :comments, each_serializer: Export::CommentSerializer
has_many :posts, each_serializer: Export::OwnPostSerializer
has_many :followed_tags
has_many :post_subscriptions
def comments
object.person.comments
has_many :relayables, each_serializer: Export::OwnRelayablesSerializer
private
def relayables
[*comments, *likes, *poll_participations]
end
%i[comments likes poll_participations].each {|collection|
delegate collection, to: :person
}
delegate :person, to: :object
def contact_groups
object.aspects
end
def private_key
object.serialized_private_key
end
def followed_tags
object.followed_tags.map(&:name)
end
def post_subscriptions
Post.subscribed_by(object).pluck(:guid)
end
end
end
# This is an ActiveModel::Serializer based class which uses DiasporaFederation::Entity JSON serialization
# features in order to serialize local DB objects. To determine a type of entity class to use the same routines
# are used as for federation messages generation.
class FederationEntitySerializer < ActiveModel::Serializer
include SerializerPostProcessing
private
def modify_serializable_object(hash)
hash.merge(entity.to_json)
end
def entity
@entity ||= Diaspora::Federation::Entities.build(object)
end
end
# This module encapsulates knowledge about the way AMS works with the serializable object.
# The main responsibility of this module is to allow changing resulting object just before the
# JSON serialization happens.
module SerializerPostProcessing
# serializable_object output is used in AMS to produce a hash from input object that is passed to JSON serializer.
# serializable_object of ActiveModel::Serializer is not documented as officialy available API
# NOTE: if we ever move to AMS 0.10, this method was renamed there to serializable_hash
def serializable_object(options={})
modify_serializable_object(super)
end
# Users of this module may override this method in order to change serializable_object after
# the serializable hash generation and before its serialization.
def modify_serializable_object(hash)
hash
end
# except is an array of keys that are excluded from serialized_object before JSON serialization
attr_accessor :except
end
......@@ -101,7 +101,8 @@ class AccountDeleter
end
def ignored_or_special_ar_person_associations
%i(comments contacts notification_actors notifications owner profile conversation_visibilities pod)
%i[comments likes poll_participations contacts notification_actors notifications owner profile
conversation_visibilities pod]
end
def mark_account_deletion_complete
......
......@@ -6,22 +6,23 @@ module Diaspora
class Exporter
SERIALIZED_VERSION = '1.0'
SERIALIZED_VERSION = "2.0".freeze
def initialize(user)
@user = user
end
def execute
@export ||= JSON.generate serialized_user.merge(version: SERIALIZED_VERSION)
JSON.generate full_archive
end
private
def serialized_user
@serialized_user ||= Export::UserSerializer.new(@user).as_json
def full_archive
{version: SERIALIZED_VERSION}
.merge(Export::UserSerializer.new(@user).as_json)
.merge(Export::OthersDataSerializer.new(@user).as_json)
end
end
end
......@@ -3,12 +3,13 @@ module Diaspora
# This class allows to query posts where a person made any activity (submitted comments,
# likes, participations or poll participations).
class PostsWithActivity
# TODO: docs
# @param user [User] user who the activity belongs to (the one who liked, commented posts, etc)
def initialize(user)
@user = user
end
# TODO: docs
# Create a request of posts with activity
# @return [Post::ActiveRecord_Relation]
def query
Post.from("(#{sql_union_all_activities}) AS posts")
end
......@@ -26,7 +27,7 @@ module Diaspora
end
def all_activities
[comments_activity, likes_activity, subscriptions, polls_activity].compact
[comments_activity, likes_activity, subscriptions, polls_activity, reshares_activity]
end
def likes_activity
......@@ -41,6 +42,10 @@ module Diaspora
other_people_posts.subscribed_by(user)
end
def reshares_activity
other_people_posts.reshared_by(person)
end
def polls_activity
StatusMessage.where.not(author_id: person.id).joins(:poll_participations)
.where(poll_participations: {author_id: person.id})
......
......@@ -50,6 +50,14 @@ module Diaspora
end
end
# Remote pods which are known to be subscribed to the post. Must include all pods which received the post in the
# past.
#
# @return [Array<String>] The list of pods' URIs
def subscribed_pods_uris
Pod.find(subscribers.select(&:remote?).map(&:pod_id).uniq).map {|pod| pod.url_to("") }
end
module QueryMethods
def owned_or_visible_by_user(user)
with_visibility.where(
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "https://diaspora.github.io/diaspora/schemas/archive_format.json",
"type": "object",
"properties": {
"user": {
"type": "object",
"properties": {
"email": { "type": "string" },
"language": { "type": "string" },
"username": { "type": "string" },
"private_key": { "type": "string" },
"disable_mail": { "type": "boolean" },
"show_community_spotlight_in_stream": { "type": "boolean" },
"auto_follow_back": { "type": "boolean" },
"auto_follow_back_aspect": {
"type": [
"string",
"null"
]
},
"strip_exif": { "type": "boolean" },
"profile": {
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#definitions/profile"
},
"contact_groups": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": { "type": "string" },
"contacts_visible": { "type": "boolean" },
"chat_enabled": { "type": "boolean" }
},
"required": [
"name"
]
}
},
"contacts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"sharing": { "type": "boolean" },
"following": { "type": "boolean" },
"receiving": { "type": "boolean" },
"followed": { "type": "boolean" },
"account_id": { "type": "string" },
"contact_groups_membership": {
"type": "array",
"items": { "type": "string" }
},
"person_name": { "type": "string" },
"person_guid": {
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/guid"
},
"public_key": { "type": "string" }
},
"required": [
"sharing",
"following",
"receiving",
"followed",
"account_id",
"contact_groups_membership"
]
}
},
"posts": {
"type": "array",
"items": {
"allOf": [
{
"$ref": "#/definitions/posts"
},
{
"oneOf": [
{ "$ref": "#/definitions/remote_subscription/public" },
{ "$ref": "#/definitions/remote_subscription/private" }
]
}
]
}
},
"relayables": {
"type": "array",
"items": {
"oneOf": [
{
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/comment"
},
{
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/like"
},
{
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/poll_participation"
}
]
}
},
"followed_tags": {
"type": "array",
"items": {
"type": "string"
}
},
"post_subscriptions": {
"type": "array",
"description": "GUIDs of posts for which changes we want to be subscribed",
"items": {
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/guid"
}
}
},
"required": [
"username",
"email",
"private_key",
"profile"
]
},
"others_data": {
"type": "object",
"properties": {
"relayables": {
"type": "array",
"items": {
"allOf": [
{
"oneOf": [
{
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/comment"
},
{
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/like"
},
{
"$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/poll_participation"
}
]
}
]
}
},
"non_contact_authors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"account_id": {
"type": "string"
},
"guid": {
"type": "string"
},
"public_key": {
"type": "string"
}
},
"required": [
"account_id",
"guid",
"public_key"
]
}
},
"posts": {
"type": "array",
"items": {
"$ref": "#/definitions/posts"
}
}
}
},
"version": {
"type": "string",
"pattern": "^2\.0$"
}
},
"required": [
"user",
"version"
],
"definitions": {
"posts": {
"oneOf": [
{ "$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/status_message" },
{ "$ref": "https://diaspora.github.io/diaspora_federation/schemas/federation_entities.json#/definitions/reshare" }
]
},
"remote_subscription": {
"public": {
"type": "object",
"properties": {
"subscribed_pods_uris": {
"type": "array",
"items": {
"type": "string"
}
},
"entity_data": {
"type": "object",
"properties": {
"public": {
"enum": [ true ]
}
},
"required": [
"public"
]
}
},
"required": [
"entity_data"
]
},
"private": {
"type": "object",
"properties": {
"subscribed_users_ids": {
"type": "array",
"items": {
"type": "string"
}
},
"entity_data": {
"type": "object",
"properties": {
"public": {
"enum": [ false ]
}
},
"required": [
"public"
]
}
}
}
}
}
}
......@@ -149,9 +149,9 @@ FactoryGirl.define do
end
factory(:location) do
address "Fernsehturm Berlin, Berlin, Germany"
lat 52.520645
lng 13.409779
sequence(:address) {|n| "Fernsehturm Berlin, #{n}, Berlin, Germany" }
sequence(:lat) {|n| 52.520645 + 0.0000001 * n }
sequence(:lng) {|n| 13.409779 + 0.0000001 * n }
end
factory :participation do
......
This diff is collapsed.
......@@ -8,7 +8,8 @@ describe Diaspora::Exporter::PostsWithActivity do
user.person.likes.first.target,
user.person.