diff --git a/app/api/unit_hub_api.rb b/app/api/unit_hub_api.rb new file mode 100644 index 0000000000..8cae88023c --- /dev/null +++ b/app/api/unit_hub_api.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +require 'grape' +require 'time' + +class UnitHubApi < Grape::API + helpers AuthenticationHelpers + + helpers do + def hub_unit! + UnitHub::Access.units_for(current_user).find(params[:unit_id]) + end + + def manageable_hub_unit! + unit = hub_unit! + error!({ error: 'Only teaching staff assigned to this unit can manage its hub.' }, 403) unless UnitHub::Access.manage?(current_user, unit) + unit + end + + def hub_attributes(key, fields, date_fields: []) + attributes = declared(params, include_missing: false).fetch(key).slice(*fields).to_h.symbolize_keys + date_fields.each do |field| + value = attributes[field] + next unless value + + unless value.match?(/T.*(?:Z|[+-]\d{2}:\d{2})\z/) + error!({ error: "#{field} must include an explicit time zone offset." }, 400) + end + attributes[field] = Time.iso8601(value) + rescue ArgumentError + error!({ error: "#{field} must be a valid ISO 8601 date and time." }, 400) + end + attributes + end + + params :announcement_fields do + requires :announcement, type: Hash do + optional :title, type: String + optional :body, type: String + optional :source_url, type: String + optional :pinned, type: Boolean + optional :published_at, type: String + optional :expires_at, type: String + end + end + + params :session_fields do + requires :session, type: Hash do + optional :title, type: String + optional :description, type: String + optional :kind, type: String, values: UnitLearningSession::KINDS + optional :start_at, type: String + optional :end_at, type: String + optional :timezone, type: String + optional :location, type: String + optional :join_url, type: String + optional :source_url, type: String + optional :published, type: Boolean + optional :cancelled, type: Boolean + optional :recurrence, type: String, values: UnitLearningSession::RECURRENCES + optional :recurrence_until, type: Date + end + end + + def announcement_attributes + hub_attributes(:announcement, %i[title body source_url pinned published_at expires_at], date_fields: %i[published_at expires_at]) + end + + def session_attributes + hub_attributes(:session, %i[title description kind start_at end_at timezone location join_url source_url published cancelled recurrence recurrence_until], date_fields: %i[start_at end_at]) + end + end + + before do + authenticated? + header 'Cache-Control', 'private, no-store' + end + + desc 'Published announcements and learning sessions for the current user units' + get '/unit_hub' do + units = UnitHub::Access.units_for(current_user).order(:code, :id).to_a + teams_configuration = UnitHub::Teams::Configuration.new + announcements = UnitAnnouncement.where(unit_id: units.map(&:id)).visible_at(Time.current).includes(:unit).recent_first.limit(101).to_a + from = 1.day.ago + to = 90.days.from_now + schedules = UnitLearningSession.where(unit_id: units.map(&:id), published: true) + .where('start_at <= ?', to) + .where('end_at >= ? OR recurrence_until >= ?', from, from.to_date) + .includes(:unit) + sessions = schedules.flat_map do |schedule| + schedule.occurrences(from: from, to: to).map { |occurrence| UnitHub::Serializer.session(schedule, occurrence) } + end + sessions.sort_by! { |occurrence| [Time.iso8601(occurrence[:start_at]), occurrence[:occurrence_id]] } + + { + units: units.map { |unit| { id: unit.id, code: unit.code, name: unit.name, can_manage: UnitHub::Access.manage?(current_user, unit), teams_sync: teams_configuration.configured_for?(unit.id) ? 'configured' : 'not_configured' } }, + announcements: announcements.first(100).map { |record| UnitHub::Serializer.announcement(record) }, + sessions: sessions, + announcements_truncated: announcements.length > 100, + window_start: from.iso8601, window_end: to.iso8601 + } + end + + resource :units do + route_param :unit_id, type: Integer do + resource :announcements do + get do + scope = manageable_hub_unit!.unit_announcements + records = scope.where(source_provider: 'manual').or(scope.visible_at(Time.current)).includes(:unit).recent_first + records.map { |record| UnitHub::Serializer.announcement(record) } + end + + params { use :announcement_fields } + post do + record = manageable_hub_unit!.unit_announcements.create!(announcement_attributes.merge(author: current_user)) + UnitHub::Serializer.announcement(record) + end + + route_param :id, type: Integer do + params { use :announcement_fields } + put do + record = manageable_hub_unit!.unit_announcements.find(params[:id]) + error!({ error: 'Manage this imported announcement in Teams.' }, 403) if record.source_provider == 'microsoft_teams' + record.update!(announcement_attributes) + UnitHub::Serializer.announcement(record) + end + + delete do + record = manageable_hub_unit!.unit_announcements.find(params[:id]) + error!({ error: 'Manage this imported announcement in Teams.' }, 403) if record.source_provider == 'microsoft_teams' + record.destroy! + { success: true } + end + end + end + + resource :sessions do + get do + records = manageable_hub_unit!.unit_learning_sessions.includes(:unit).order(:start_at, :id) + records.map { |record| UnitHub::Serializer.session(record) } + end + + params { use :session_fields } + post do + record = manageable_hub_unit!.unit_learning_sessions.create!(session_attributes.merge(author: current_user)) + UnitHub::Serializer.session(record) + end + + route_param :id, type: Integer do + params { use :session_fields } + put do + record = manageable_hub_unit!.unit_learning_sessions.find(params[:id]) + record.update!(session_attributes) + UnitHub::Serializer.session(record) + end + + delete do + # Preserve the UID and schedule so subscribed calendars receive an + # explicit cancellation instead of retaining an old working join link. + record = manageable_hub_unit!.unit_learning_sessions.find(params[:id]) + record.update!(cancelled: true) + { success: true } + end + end + end + end + end +end diff --git a/app/models/concerns/unit_hub_links.rb b/app/models/concerns/unit_hub_links.rb new file mode 100644 index 0000000000..50eeacf8c7 --- /dev/null +++ b/app/models/concerns/unit_hub_links.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'uri' + +# Links are displayed, never fetched by the server. Reject executable protocols +# and embedded credentials before staff can publish a link. +module UnitHubLinks + extend ActiveSupport::Concern + + included do + validate :safe_unit_hub_links + end + + private + + def safe_unit_hub_links + %i[source_url join_url].each do |field| + next unless respond_to?(field) + + value = public_send(field) + next if value.blank? + + valid = value.length <= 2048 && !value.match?(/[[:space:][:cntrl:]]/) + uri = URI.parse(value) if valid + valid &&= uri.is_a?(URI::HTTPS) && uri.host.present? && uri.userinfo.nil? + errors.add(field, 'must be a complete HTTPS link without embedded credentials') unless valid + rescue URI::InvalidURIError + errors.add(field, 'must be a complete HTTPS link without embedded credentials') + end + end +end diff --git a/app/models/teams_announcement_sync_state.rb b/app/models/teams_announcement_sync_state.rb new file mode 100644 index 0000000000..22d71f002a --- /dev/null +++ b/app/models/teams_announcement_sync_state.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class TeamsAnnouncementSyncState < ApplicationRecord + belongs_to :unit + validates :mapping_key, presence: true + validates :status, inclusion: { in: %w[pending synced failed throttled] } +end diff --git a/app/models/unit_announcement.rb b/app/models/unit_announcement.rb new file mode 100644 index 0000000000..bbd62f848e --- /dev/null +++ b/app/models/unit_announcement.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class UnitAnnouncement < ApplicationRecord + include UnitHubLinks + + belongs_to :unit + belongs_to :author, class_name: 'User', optional: true + + validates :title, presence: true, length: { maximum: 200 } + validates :body, presence: true, length: { maximum: 20_000 } + validates :pinned, inclusion: { in: [true, false] } + validates :source_provider, inclusion: { in: %w[manual microsoft_teams] } + validate :expiry_follows_publication + + scope :allowed_sources, lambda { + where(source_provider: 'manual').or(where(source_provider: 'microsoft_teams', source_mapping_key: UnitHub::Teams::Configuration.new.visible_mapping_keys)) + } + scope :visible_at, lambda { |at| + allowed_sources.where('published_at <= ?', at).where('expires_at IS NULL OR expires_at > ?', at) + } + scope :recent_first, -> { order(pinned: :desc, published_at: :desc, id: :desc) } + + # After commit, so the job that fans out never runs before the row it reads + # is visible, and a rolled back save tells nobody anything. + after_commit(on: :create) { queue_hub_notifications(created: true) } + after_commit(on: :update) { queue_hub_notifications(created: false) } + + private + + # A notification must never stop an announcement being saved. + def queue_hub_notifications(created:) + UnitHub::Notifications.announcement_committed(self, created: created) + rescue StandardError => e + Rails.logger.error("Failed to queue Unit Hub notifications for UnitAnnouncement #{id}: #{e.class}") + end + + def expiry_follows_publication + return unless published_at && expires_at && expires_at <= published_at + + errors.add(:expires_at, 'must be after publication') + end +end diff --git a/app/models/unit_learning_session.rb b/app/models/unit_learning_session.rb new file mode 100644 index 0000000000..06b1512718 --- /dev/null +++ b/app/models/unit_learning_session.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +class UnitLearningSession < ApplicationRecord + include UnitHubLinks + + KINDS = %w[helphub lecture seminar workshop other].freeze + RECURRENCES = %w[none weekly].freeze + + belongs_to :unit + belongs_to :author, class_name: 'User', optional: true + + validates :title, presence: true, length: { maximum: 200 } + validates :description, length: { maximum: 20_000 } + validates :location, length: { maximum: 300 } + validates :kind, inclusion: { in: KINDS } + validates :recurrence, inclusion: { in: RECURRENCES } + validates :start_at, :end_at, :timezone, presence: true + validates :published, :cancelled, inclusion: { in: [true, false] } + validate :valid_schedule + + # Only updates: a new session is not one of the changes people are told + # about, and the delete endpoint cancels rather than destroys. + after_commit :queue_hub_notifications, on: :update + + # Add calendar weeks in the named zone, not 604800 seconds in UTC: the local + # HelpHub time stays constant across daylight saving changes. + def occurrences(from:, to:) + return [] unless start_at && end_at + + local_start = start_at.in_time_zone(timezone) + local_end = end_at.in_time_zone(timezone) + result = [] + 27.times do |index| + occurrence_start = local_start + index.weeks + occurrence_end = local_end + index.weeks + break if index.positive? && recurrence != 'weekly' + break if recurrence == 'weekly' && occurrence_start.to_date > recurrence_until + break if occurrence_start > to + + if occurrence_end >= from + result << { occurrence_id: "#{id}-#{index}", start_at: occurrence_start, end_at: occurrence_end } + end + end + result + end + + private + + # A notification must never stop a session being saved. + def queue_hub_notifications + UnitHub::Notifications.session_committed(self) + rescue StandardError => e + Rails.logger.error("Failed to queue Unit Hub notifications for UnitLearningSession #{id}: #{e.class}") + end + + def valid_schedule + begin + TZInfo::Timezone.get(timezone.to_s) + rescue TZInfo::InvalidTimezoneIdentifier + errors.add(:timezone, 'must be an IANA time zone such as Australia/Melbourne') + return + end + return unless start_at && end_at + + errors.add(:end_at, 'must be after the start and within 24 hours') unless end_at > start_at && end_at <= start_at + 24.hours + return unless recurrence == 'weekly' + + first_date = start_at.in_time_zone(timezone).to_date + unless recurrence_until && recurrence_until >= first_date && recurrence_until <= first_date + 6.months + errors.add(:recurrence_until, 'must be on or after the first session and within six months') + end + end +end diff --git a/app/services/unit_hub/access.rb b/app/services/unit_hub/access.rb new file mode 100644 index 0000000000..63d23df9eb --- /dev/null +++ b/app/services/unit_hub/access.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module UnitHub + # Authorisation is always derived from current enrolments and assigned unit + # roles. A global staff role alone never opens an unrelated unit's content. + class Access + def self.units_for(user) + student_ids = Project.where(user_id: user.id, enrolled: true).select(:unit_id) + staff_ids = UnitRole.where(user_id: user.id, role_id: [Role.tutor.id, Role.convenor.id]).select(:unit_id) + Unit.where(active: true).where(id: student_ids).or(Unit.where(active: true, id: staff_ids)) + end + + def self.manage?(user, unit) + UnitRole.exists?(user_id: user.id, unit_id: unit.id, + role_id: [Role.tutor.id, Role.convenor.id], observer_only: false) + end + end +end diff --git a/app/services/unit_hub/notifications.rb b/app/services/unit_hub/notifications.rb new file mode 100644 index 0000000000..a6204d1b59 --- /dev/null +++ b/app/services/unit_hub/notifications.rb @@ -0,0 +1,294 @@ +# frozen_string_literal: true + +require 'digest' + +module UnitHub + # Notifications about Unit Hub announcements and learning sessions. + # + # The models call the *_committed methods from after_commit, which decide + # whether a save is worth telling anyone about and queue a job with only ids + # and a version. The jobs call fan_out, which walks the unit's recipients in + # batches and raises each notification through NotificationService, so the + # category preference, email and push all behave like every other event. + # + # Every notification carries a dedupe key made of the event, the record and a + # version of it, which the unique index scopes to the recipient. A retried or + # duplicated job finds the row that is already there instead of sending again. + module Notifications + TYPE = 'unit_hub' + + ANNOUNCEMENT_PUBLISHED = 'unit_announcement_published' + ANNOUNCEMENT_UPDATED = 'unit_announcement_updated' + SESSION_CHANGED = 'unit_session_changed' + SESSION_STARTING_SOON = 'unit_session_starting_soon' + EVENTS = [ANNOUNCEMENT_PUBLISHED, ANNOUNCEMENT_UPDATED, SESSION_CHANGED, SESSION_STARTING_SOON].freeze + + BATCH_SIZE = 200 + + # At most one update notification per announcement per recipient in this + # window. A publish inside the window counts, so fixing a line straight + # after posting does not send a second notification. + UPDATE_DEBOUNCE = 30.minutes + + # How long before a session its reminder goes out. + REMINDER_LEAD = 30.minutes + + # An announcement created with a publication time older than this is an + # import or a backfill, not news, so it does not raise a publish. + BACKFILL_AGE = 1.day + + SESSION_SCHEDULE_FIELDS = %w[start_at end_at timezone recurrence recurrence_until location join_url].freeze + + # A one letter fix in a word of four or more letters is a typo. Anything + # else that changes the words, or any change to a number, is not. + TYPO_DISTANCE = 2 + + module_function + + # ---- deciding what a save means ------------------------------------------ + + def announcement_committed(record, created:) + now = Time.current + return unless announcement_source_allowed?(record) + + if created + return if record.published_at.nil? + return if record.published_at < record.created_at - BACKFILL_AGE + + return queue_publish(record, now) + end + + changes = record.saved_changes + was_visible = visible_with?( + changes.key?('published_at') ? changes['published_at'].first : record.published_at, + changes.key?('expires_at') ? changes['expires_at'].first : record.expires_at, + now + ) + + unless was_visible + return unless changes.keys.intersect?(%w[published_at expires_at]) + return if record.published_at.nil? || (record.expires_at && record.expires_at <= now) + + return queue_publish(record, now) + end + return unless visible_with?(record.published_at, record.expires_at, now) + + # Pinning a visible announcement puts it back in front of people, so it is + # raised as a publish with its own version. Unpinning is not news. + if changes.key?('pinned') && record.pinned + UnitAnnouncementNotificationJob.perform_async(record.id, ANNOUNCEMENT_PUBLISHED, "pinned-#{record.updated_at.to_i}") + return + end + + old_title = changes.key?('title') ? changes['title'].first : record.title + old_body = changes.key?('body') ? changes['body'].first : record.body + return unless meaningful_edit?(old_title, record.title) || meaningful_edit?(old_body, record.body) + + UnitAnnouncementNotificationJob.perform_async(record.id, ANNOUNCEMENT_UPDATED, nil) + end + + def session_committed(record) + changes = record.saved_changes + published_before = changes.key?('published') ? changes['published'].first : record.published + return unless record.published && published_before + + cancelled_now = changes.key?('cancelled') && record.cancelled + restored = changes.key?('cancelled') && !record.cancelled + schedule_changed = changes.keys.intersect?(SESSION_SCHEDULE_FIELDS) + return unless cancelled_now || (!record.cancelled && (schedule_changed || restored)) + return if next_occurrence(record, Time.current).nil? + + UnitSessionNotificationJob.perform_async(record.id, record.lock_version, cancelled_now ? 'cancelled' : 'changed') + end + + def queue_publish(record, now) + version = "published-#{record.published_at.to_i}" + if record.published_at > now + UnitAnnouncementNotificationJob.perform_at(record.published_at, record.id, ANNOUNCEMENT_PUBLISHED, version) + else + UnitAnnouncementNotificationJob.perform_async(record.id, ANNOUNCEMENT_PUBLISHED, version) + end + end + + def visible_with?(published_at, expires_at, at) + published_at.present? && published_at <= at && (expires_at.nil? || expires_at > at) + end + + def announcement_source_allowed?(record) + UnitAnnouncement.allowed_sources.exists?(id: record.id) + end + + # Whether an edit changes what an announcement says, rather than fixing how + # it is spelled. Case, spacing and punctuation never count. A number always + # counts, because a changed date, time or room is the edit people need to + # hear about. One word swapped for a near spelling, or a doubled word taken + # out, is a typo. Any other change to the words counts. + def meaningful_edit?(before, after) + old_words = words(before) + new_words = words(after) + removed = multiset_difference(old_words, new_words) + added = multiset_difference(new_words, old_words) + changed = removed + added + + return false if changed.empty? + return true if changed.any? { |word| word.match?(/\d/) } + + if removed.length == 1 && added.length == 1 + return true if [removed.first.length, added.first.length].min < 4 + + return DidYouMean::Levenshtein.distance(removed.first, added.first) > TYPO_DISTANCE + end + + return !(old_words.include?(changed.first) && new_words.include?(changed.first)) if changed.length == 1 + + true + end + + def words(text) + text.to_s.downcase.scan(/[[:alnum:]]+/) + end + + def multiset_difference(left, right) + remaining = right.tally + left.each_with_object([]) do |word, result| + if remaining[word].to_i.positive? + remaining[word] -= 1 + else + result << word + end + end + end + + # ---- who hears about it -------------------------------------------------- + + # Enrolled students and teaching staff of an active unit, with the Unit Hub + # category on, never the person who wrote the thing. + def recipients(unit, author_id: nil) + return User.none unless unit&.active + + student_ids = Project.where(unit_id: unit.id, enrolled: true).select(:user_id) + staff_ids = UnitRole.where(unit_id: unit.id, role_id: [Role.tutor.id, Role.convenor.id]).select(:user_id) + scope = User.where(id: student_ids).or(User.where(id: staff_ids)).where(receive_unit_hub_notifications: true) + author_id ? scope.where.not(id: author_id) : scope + end + + # Raise one notification per recipient, a batch at a time. + # + # skip - called with a batch of user ids, returns the ids to leave out on + # top of the ones that already hold this dedupe key. + # Failures are collected and raised at the end so Sidekiq retries the whole + # fan-out, which the dedupe key makes safe. + def fan_out(scope:, event:, dedupe_key:, notifiable:, message:, link:, skip: nil) + failed = [] + scope.find_in_batches(batch_size: BATCH_SIZE) do |batch| + ids = batch.map(&:id) + skipped = Notification.where(user_id: ids, dedupe_key: dedupe_key).pluck(:user_id) + skipped.concat(skip.call(ids)) if skip + skipped = skipped.to_set + + batch.each do |user| + next if skipped.include?(user.id) + + NotificationService.notify( + user: user, type: TYPE, event: event, message: message, + link: link, dedupe_key: dedupe_key, notifiable: notifiable + ) + rescue StandardError => e + failed << user.id + Rails.logger.error("Failed #{event} notification for User #{user.id}: #{e.class}") + end + end + + raise "#{event} notifications failed for users: #{failed.join(', ')}" if failed.any? + end + + # ---- what it says -------------------------------------------------------- + + def announcement_link(record) + "/unit-hub?unit=#{record.unit_id}&announcement=#{record.id}" + end + + def session_link(record) + "/unit-hub?unit=#{record.unit_id}&session=#{record.id}" + end + + def announcement_message(record, event) + prefix = + if event == ANNOUNCEMENT_UPDATED + 'Announcement updated' + elsif record.pinned + 'Pinned announcement' + else + 'New announcement' + end + "#{prefix} in #{record.unit.code}: #{record.title}".truncate(500) + end + + def session_changed_message(record, change, at: Time.current) + unit_code = record.unit.code + if change == 'cancelled' + return "The weekly #{record.title} sessions in #{unit_code} are cancelled.".truncate(500) if record.recurrence == 'weekly' + + occurrence = next_occurrence(record, at) + when_text = occurrence ? " on #{format_time(occurrence[:start_at], record.timezone)}" : '' + return "#{record.title} in #{unit_code}#{when_text} is cancelled.".truncate(500) + end + + "#{record.title} in #{unit_code} has changed. #{where_and_when(record, at)}".truncate(500) + end + + def session_starting_soon_message(record, start_at) + place = session_place(record) + "#{record.title} in #{record.unit.code} starts at #{format_clock(start_at, record.timezone)}#{" (#{place})" if place}.".truncate(500) + end + + def where_and_when(record, at) + occurrence = next_occurrence(record, at) + return 'Open the Unit Hub for the details.' if occurrence.nil? + + cadence = record.recurrence == 'weekly' ? 'Next session' : 'Now' + place = session_place(record) + "#{cadence}: #{format_time(occurrence[:start_at], record.timezone)}#{", #{place}" if place}." + end + + def session_place(record) + return record.location if record.location.present? + + 'online' if record.join_url.present? + end + + def next_occurrence(record, at) + record.occurrences(from: at, to: at + 7.months).find { |occurrence| occurrence[:end_at] >= at } + end + + def format_time(time, zone) + local = time.in_time_zone(zone) + "#{local.strftime('%a %-d %b')} at #{format_clock(local, zone)}" + end + + def format_clock(time, zone) + local = time.in_time_zone(zone) + "#{local.strftime('%-l:%M%P')} #{local.zone}" + end + + # The details an email shows for a session, in the session's own zone. + def session_details(record, at:) + occurrence = next_occurrence(record, at) + details = { 'Unit' => "#{record.unit.code} #{record.unit.name}", 'Session' => record.title } + if occurrence + local_start = occurrence[:start_at].in_time_zone(record.timezone) + local_end = occurrence[:end_at].in_time_zone(record.timezone) + details['When'] = "#{format_time(local_start, record.timezone)} to #{local_end.strftime('%-l:%M%P')}" + details['Repeats'] = "Weekly until #{record.recurrence_until.strftime('%-d %b %Y')}" if record.recurrence == 'weekly' && record.recurrence_until + end + details['Where'] = record.location if record.location.present? + details['Online'] = 'A join link is on the Unit Hub' if record.join_url.present? && !record.cancelled + details['Status'] = 'Cancelled' if record.cancelled + details + end + + def announcement_version(record) + Digest::SHA256.hexdigest([record.title, record.body].join(""))[0, 16] + end + end +end diff --git a/app/services/unit_hub/serializer.rb b/app/services/unit_hub/serializer.rb new file mode 100644 index 0000000000..b4412b57a5 --- /dev/null +++ b/app/services/unit_hub/serializer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module UnitHub + class Serializer + def self.announcement(record) + record.attributes.slice('id', 'unit_id', 'title', 'body', 'source_url', 'pinned').symbolize_keys.merge( + unit_code: record.unit.code, unit_name: record.unit.name, + published_at: record.published_at&.iso8601, expires_at: record.expires_at&.iso8601, + updated_at: record.updated_at.iso8601, source_provider: record.source_provider, + managed_externally: record.source_provider == 'microsoft_teams', + author_name: record.source_provider == 'microsoft_teams' ? 'Teaching team' : nil, + source_imported_at: record.source_imported_at&.iso8601 + ) + end + + def self.session(record, occurrence = nil) + values = record.attributes.slice('id', 'unit_id', 'title', 'description', 'kind', 'timezone', + 'location', 'join_url', 'source_url', 'published', 'cancelled', + 'recurrence').symbolize_keys + values.merge!(unit_code: record.unit.code, unit_name: record.unit.name, + start_at: record.start_at.iso8601, end_at: record.end_at.iso8601, + recurrence_until: record.recurrence_until&.iso8601, updated_at: record.updated_at.iso8601) + if occurrence + values.merge!(occurrence_id: occurrence[:occurrence_id], original_start_at: record.start_at.iso8601, + start_at: occurrence[:start_at].iso8601, end_at: occurrence[:end_at].iso8601) + values[:join_url] = nil if record.cancelled? + end + values + end + end +end diff --git a/app/services/unit_hub/teams/announcement_sync.rb b/app/services/unit_hub/teams/announcement_sync.rb new file mode 100644 index 0000000000..e6ca0d42c4 --- /dev/null +++ b/app/services/unit_hub/teams/announcement_sync.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'cgi' +require 'digest' +require 'time' + +module UnitHub + module Teams + class AnnouncementSync + SCAN_LIMIT = 25 + SNAPSHOT_LIFETIME = 7.days + SOURCE_HOSTS = %w[teams.microsoft.com teams.cloud.microsoft].freeze + + def initialize(configuration: Configuration.new, client: nil, now: Time.current) + @configuration = configuration + @client = client + @now = now + end + + def call + return { status: 'disabled', synced: 0, failed: 0 } unless @configuration.enabled? + + mappings = @configuration.mappings + @client ||= GraphClient.new(@configuration.credentials) + result = { status: 'complete', synced: 0, failed: 0 } + mappings.rotate((@now.to_i / 300) % mappings.length).each do |mapping| + unit = Unit.find_by(id: mapping.unit_id, active: true) + next unless unit + + state = TeamsAnnouncementSyncState.create_or_find_by!(mapping_key: mapping.key) { |row| row.unit = unit } + next if state.next_attempt_at && state.next_attempt_at > @now + + state.update!(last_attempt_at: @now) + sync_mapping(mapping, unit) + state.update!(status: 'synced', last_succeeded_at: @now, next_attempt_at: nil) + result[:synced] += 1 + rescue GraphClient::Throttled => e + # Persist the provider cooldown across worker processes and cron runs. + mappings.each do |entry| + next unless Unit.exists?(id: entry.unit_id, active: true) + + checkpoint = TeamsAnnouncementSyncState.create_or_find_by!(mapping_key: entry.key) { |row| row.unit_id = entry.unit_id } + checkpoint.update!(status: 'throttled', next_attempt_at: @now + e.retry_after.seconds) + end + result[:status] = 'throttled' + break + rescue GraphClient::Error => e + # Revoke visibility atomically even if a provider copy later fails validation. + # rubocop:disable Rails/SkipsModelValidations + imported_for(mapping).update_all(published_at: nil) if e.is_a?(GraphClient::AccessDenied) || e.is_a?(GraphClient::MissingMessage) + # rubocop:enable Rails/SkipsModelValidations + state.update!(status: 'failed') + result[:failed] += 1 + Rails.logger.warn('Teams announcement sync failed for a configured mapping.') + end + result + end + + private + + def sync_mapping(mapping, unit) + @client.messages(mapping).each { |message| import_message(mapping, unit, message) } + UnitAnnouncement.where(unit_id: unit.id, source_provider: 'microsoft_teams', source_channel_key: channel_key(mapping)) + .where('source_checked_at IS NULL OR source_checked_at < ?', @now) + .order(:source_checked_at, :id).limit(SCAN_LIMIT).each do |record| + message = @client.message(mapping, record.external_message_id) + unless message['id'] == record.external_message_id + raise GraphClient::Error, 'Teams returned an unexpected message.' + end + import_message(mapping, unit, message) + rescue GraphClient::MissingMessage + record.update!(published_at: nil, source_checked_at: @now) + end + end + + def imported_for(mapping) + UnitAnnouncement.where(unit_id: mapping.unit_id, source_provider: 'microsoft_teams', source_mapping_key: mapping.key) + end + + def import_message(mapping, unit, message) + return unless message.is_a?(Hash) && message['id'].is_a?(String) && message['id'].match?(GraphClient::MESSAGE_ID) + + key = Digest::SHA256.hexdigest(JSON.generate([@client.tenant_id, mapping.team_id, mapping.channel_id, message['id']])) + record = unit.unit_announcements.find_or_initialize_by(external_source_key: key) + return if record.persisted? && record.source_provider != 'microsoft_teams' + + unless approved_message?(mapping, message) && message['deletedDateTime'].blank? + record.update!(published_at: nil, source_checked_at: @now) if record.persisted? + return + end + source_url = source_url(message['webUrl'], mapping, message['id']) + unless source_url + record.update!(published_at: nil, source_checked_at: @now) if record.persisted? + return + end + updated_at = Time.iso8601(message['lastModifiedDateTime'] || message.fetch('createdDateTime')) + # Serialize source-version comparison and update for overlapping cron + # and operator runs. New-record uniqueness is enforced by the database. + record.with_lock do + if record.source_updated_at && record.source_updated_at > updated_at + record.update!(source_checked_at: @now) + else + body = plain_text(message.dig('body', 'content')) + body = 'Open the original announcement in Teams for attached content.' if body.blank? + subject = plain_text(message['subject']) + title = (subject.presence || body.lines.first.presence || 'Unit announcement').strip.first(200) + record.assign_attributes(source_provider: 'microsoft_teams', external_message_id: message['id'], + source_mapping_key: mapping.key, source_channel_key: channel_key(mapping), source_updated_at: updated_at, + source_imported_at: @now, source_checked_at: @now, + title: title, body: body.first(20_000).byteslice(0, 60_000).scrub, source_url: source_url, author_id: nil, + pinned: false, published_at: Time.iso8601(message.fetch('createdDateTime')), + expires_at: @now + SNAPSHOT_LIFETIME) + record.save! + end + end + rescue ActiveRecord::RecordNotUnique + attempts ||= 0 + attempts += 1 + retry if attempts <= 1 + + Rails.logger.warn('Teams announcement was concurrently imported; it will be checked next run.') + rescue ArgumentError, KeyError, TypeError, ActiveRecord::RecordInvalid + # Provider content is never copied into diagnostics. + Rails.logger.warn('Skipped an invalid Teams announcement.') + end + + def approved_message?(mapping, message) + identity = message['channelIdentity'] + return false unless identity.is_a?(Hash) && identity['teamId'].to_s.downcase == mapping.team_id && identity['channelId'] == mapping.channel_id + + sender = message['from'] + sender.is_a?(Hash) && sender['application'].blank? && sender['user'].is_a?(Hash) && + mapping.publisher_ids.include?(sender['user']['id'].to_s.downcase) && + message['messageType'] == 'message' && message['replyToId'].blank? + end + + def channel_key(mapping) + Digest::SHA256.hexdigest(JSON.generate([@client.tenant_id, mapping.team_id, mapping.channel_id])) + end + + def source_url(value, mapping, message_id) + return nil unless value.is_a?(String) && value.bytesize <= 2048 && !value.match?(/[[:space:][:cntrl:]]/) + + uri = URI.parse(value) + return nil unless uri.is_a?(URI::HTTPS) && SOURCE_HOSTS.include?(uri.host) && uri.port == 443 && uri.userinfo.nil? && uri.fragment.nil? + return nil unless URI::DEFAULT_PARSER.unescape(uri.path) == "/l/message/#{mapping.channel_id}/#{message_id}" + + query = URI.decode_www_form(uri.query.to_s).group_by(&:first) + return nil unless query['groupId']&.length == 1 && query['tenantId']&.length == 1 + return nil unless query['groupId'].first.last.downcase == mapping.team_id && query['tenantId'].first.last.downcase == @client.tenant_id + + "https://teams.microsoft.com/l/message/#{ERB::Util.url_encode(mapping.channel_id)}/#{message_id}?#{URI.encode_www_form(groupId: mapping.team_id, tenantId: @client.tenant_id)}" + rescue URI::InvalidURIError + nil + end + + def plain_text(value) + html = value.to_s.gsub(%r{|}i, "\n") + CGI.unescapeHTML(ActionController::Base.helpers.sanitize(html, tags: [], attributes: []).to_s) + .gsub(/\r\n?/, "\n").strip + end + end + end +end diff --git a/app/services/unit_hub/teams/configuration.rb b/app/services/unit_hub/teams/configuration.rb new file mode 100644 index 0000000000..0b672fb30f --- /dev/null +++ b/app/services/unit_hub/teams/configuration.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'digest' +require 'json' + +module UnitHub + module Teams + class Configuration + class Error < StandardError; end + GUID = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i + CHANNEL_ID = /\A19:[A-Za-z0-9_-]+@thread\.(?:tacv2|skype)\z/ + Mapping = Struct.new(:unit_id, :team_id, :channel_id, :publisher_ids, :key, keyword_init: true) + + def initialize(env = ENV) + @env = env + end + + def enabled? + @env['DF_TEAMS_ANNOUNCEMENTS_ENABLED'] == 'true' + end + + # Visibility configuration intentionally needs no Graph client secret. + # Only the worker has access to application credentials. + def mappings + return [] unless enabled? + + @mappings ||= parse_mappings + end + + def visible_mapping_keys + mappings.map(&:key) + rescue Error + [] + end + + def configured_for?(unit_id) + mappings.any? { |mapping| mapping.unit_id == unit_id } + rescue Error + false + end + + def credentials + tenant_id = @env['DF_TEAMS_TENANT_ID'].to_s + client_id = @env['DF_TEAMS_CLIENT_ID'].to_s + secret = @env['DF_TEAMS_CLIENT_SECRET'].to_s + unless tenant_id.match?(GUID) && client_id.match?(GUID) && secret.present? && secret.bytesize <= 4096 + raise Error, 'Teams application credentials are missing or invalid.' + end + + { tenant_id: tenant_id.downcase, client_id: client_id.downcase, client_secret: secret } + end + + private + + def parse_mappings + raw = @env['DF_TEAMS_CHANNEL_MAPPINGS'].to_s + raise Error, 'Teams channel mappings are missing or invalid.' if raw.bytesize > 100_000 + + unless @env['DF_TEAMS_TENANT_ID'].to_s.match?(GUID) + raise Error, 'Teams tenant scope is missing or invalid.' + end + values = JSON.parse(raw) + unless values.is_a?(Array) && values.length.between?(1, 20) + raise Error, 'Configure between one and twenty Teams channel mappings.' + end + result = values.map { |value| parse_mapping(value) } + identities = result.map { |mapping| [mapping.unit_id, mapping.team_id, mapping.channel_id] } + raise Error, 'Duplicate Teams channel mapping.' unless identities.uniq.length == identities.length + + result + rescue JSON::ParserError + raise Error, 'Teams channel mappings are not valid JSON.' + end + + def parse_mapping(value) + unless value.is_a?(Hash) && value['student_visible'] == true && + value['unit_id'].is_a?(Integer) && value['unit_id'].positive? && + value['team_id'].is_a?(String) && value['team_id'].match?(GUID) && + value['channel_id'].is_a?(String) && value['channel_id'].length <= 256 && value['channel_id'].match?(CHANNEL_ID) + raise Error, 'Each mapping must identify a student-visible unit channel.' + end + publishers = value['publisher_ids'] + unless publishers.is_a?(Array) && publishers.length.between?(1, 100) && publishers.all? { |id| id.is_a?(String) && id.match?(GUID) } + raise Error, 'Each mapping needs an allowlist of approved staff publisher IDs.' + end + publishers = publishers.map(&:downcase).uniq.sort + identity = [@env['DF_TEAMS_TENANT_ID'].downcase, value['unit_id'], value['team_id'].downcase, value['channel_id'], true, publishers] + Mapping.new(unit_id: value['unit_id'], team_id: value['team_id'].downcase, + channel_id: value['channel_id'], publisher_ids: publishers, + key: Digest::SHA256.hexdigest(JSON.generate(identity))) + end + end + end +end diff --git a/app/services/unit_hub/teams/graph_client.rb b/app/services/unit_hub/teams/graph_client.rb new file mode 100644 index 0000000000..c6b554752b --- /dev/null +++ b/app/services/unit_hub/teams/graph_client.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'net/http' +require 'uri' +require 'json' +require 'timeout' + +module UnitHub + module Teams + class GraphClient + class Error < StandardError; end + class AccessDenied < Error; end + class MissingMessage < Error; end + + class Throttled < Error + attr_reader :retry_after + + def initialize(retry_after) + @retry_after = retry_after.to_i.clamp(60, 86_400) + super('Teams is throttling requests.') + end + end + MAX_RESPONSE_BYTES = 2.megabytes + MAX_PAGES = 2 + MESSAGE_ID = /\A[A-Za-z0-9_-]{1,128}\z/ + + attr_reader :tenant_id + + def initialize(credentials, deadline: Process.clock_gettime(Process::CLOCK_MONOTONIC) + 180) + @credentials = credentials + @tenant_id = credentials.fetch(:tenant_id) + unless @tenant_id.match?(Configuration::GUID) && credentials.fetch(:client_id).match?(Configuration::GUID) + raise Error, 'Teams application identifiers are invalid.' + end + @deadline = deadline + end + + def messages(mapping) + path = channel_path(mapping) + url = "https://graph.microsoft.com#{path}?$top=50" + visited = [] + result = [] + MAX_PAGES.times do + raise Error, 'Teams pagination repeated a page.' if visited.include?(url) + + visited << url + response = get_json(url, path: path) + values = response['value'] + raise Error, 'Teams returned an invalid message collection.' unless values.is_a?(Array) && values.length <= 50 + + result.concat(values) + url = response['@odata.nextLink'] + break if url.blank? + + validate_graph_url!(url, path) + end + result + end + + def message(mapping, id) + raise Error, 'Teams message ID is invalid.' unless id.to_s.match?(MESSAGE_ID) + + path = "#{channel_path(mapping)}/#{id}" + get_json("https://graph.microsoft.com#{path}", path: path) + end + + private + + def channel_path(mapping) + "/v1.0/teams/#{mapping.team_id}/channels/#{ERB::Util.url_encode(mapping.channel_id)}/messages" + end + + def validate_graph_url!(value, expected_path) + uri = URI.parse(value.to_s) + unless value.is_a?(String) && value.bytesize <= 8192 && !value.match?(/[[:space:][:cntrl:]]/) && + uri.is_a?(URI::HTTPS) && uri.host == 'graph.microsoft.com' && uri.port == 443 && + uri.userinfo.nil? && uri.fragment.nil? && + URI::DEFAULT_PARSER.unescape(uri.path) == URI::DEFAULT_PARSER.unescape(expected_path) + raise Error, 'Teams pagination URL is outside the configured channel.' + end + uri + rescue URI::InvalidURIError + raise Error, 'Teams pagination URL is invalid.' + end + + def token + return @token if @token + + uri = URI("https://login.microsoftonline.com/#{@tenant_id}/oauth2/v2.0/token") + request = Net::HTTP::Post.new(uri) + request.set_form_data(client_id: @credentials.fetch(:client_id), client_secret: @credentials.fetch(:client_secret), + grant_type: 'client_credentials', scope: 'https://graph.microsoft.com/.default') + response = request_json(uri, request, token_request: true) + access_token = response['access_token'] + unless response['token_type'].to_s.casecmp('Bearer').zero? && access_token.is_a?(String) && + access_token.bytesize.between?(1, 32_768) && !access_token.match?(/[[:space:][:cntrl:]]/) + raise Error, 'Teams token response was invalid.' + end + @token = access_token + end + + def get_json(url, path:) + uri = validate_graph_url!(url, path) + request = Net::HTTP::Get.new(uri) + request['Authorization'] = "Bearer #{token}" + request['Accept'] = 'application/json' + request_json(uri, request) + end + + def request_json(uri, request, token_request: false) + raise Error, 'Teams sync time limit reached.' if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @deadline + + body = +'' + code = nil + retry_after = 300 + Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, open_timeout: 5, read_timeout: 10, write_timeout: 5) do |http| + http.max_retries = 0 + http.request(request) do |response| + code = response.code.to_i + retry_after = response['Retry-After'].to_i if response['Retry-After'].to_s.match?(/\A\d+\z/) + response.read_body do |chunk| + raise Error, 'Teams sync time limit reached.' if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= @deadline + raise Error, 'Teams response exceeded the size limit.' if body.bytesize + chunk.bytesize > MAX_RESPONSE_BYTES + + body << chunk + end + end + end + raise Throttled, retry_after if code == 429 + raise AccessDenied, 'Teams access was denied.' if [401, 403].include?(code) || (token_request && code == 400) + raise MissingMessage, 'Teams message is unavailable.' if [404, 410].include?(code) + raise Error, "Teams request failed (HTTP #{code})." unless code == 200 + + value = JSON.parse(body) + raise Error, 'Teams returned an invalid response.' unless value.is_a?(Hash) + + value + rescue JSON::ParserError + raise Error, 'Teams returned invalid JSON.' + rescue IOError, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::ProtocolError + # Never propagate provider bodies, URLs, request headers or credentials. + raise Error, 'Teams could not be reached securely.' + end + end + end +end diff --git a/app/sidekiq/send_unit_session_reminders_job.rb b/app/sidekiq/send_unit_session_reminders_job.rb new file mode 100644 index 0000000000..873339124b --- /dev/null +++ b/app/sidekiq/send_unit_session_reminders_job.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# Remind people who opted in that a Unit Hub session starts soon. +# +# Nobody saves anything when a session approaches, so this is swept for on a +# schedule like SendDueSoonRemindersJob. config/schedule.yml runs it every five +# minutes. Each run looks for occurrences starting within REMINDER_LEAD, so one +# occurrence is seen by several runs. The dedupe key names the occurrence by +# its start time, which makes every run after the first find the row already +# there, and a run that is late still reminds anyone before the start. +class SendUnitSessionRemindersJob + include Sidekiq::Job + + sidekiq_options queue: :notifications, + lock: :until_executed, + lock_args_method: ->(_args) { ['send-unit-session-reminders'] }, + on_conflict: :reject, + retry: 1 + + def perform + now = Time.current + horizon = now + UnitHub::Notifications::REMINDER_LEAD + failures = [] + + candidates(now, horizon).find_each do |session| + session.occurrences(from: now, to: horizon).each do |occurrence| + next unless occurrence[:start_at] > now && occurrence[:start_at] <= horizon + + remind(session, occurrence[:start_at]) + end + rescue StandardError => e + failures << session.id + Rails.logger.error("Failed session reminders for UnitLearningSession #{session.id}: #{e.class}") + end + + raise "Session reminders failed for sessions: #{failures.join(', ')}" if failures.any? + end + + private + + def candidates(now, horizon) + UnitLearningSession.joins(:unit) + .where(units: { active: true }, published: true, cancelled: false) + .where(start_at: ..horizon) + .where('unit_learning_sessions.end_at >= ? OR unit_learning_sessions.recurrence_until >= ?', now, now.to_date - 1) + .includes(:unit) + end + + def remind(session, start_at) + event = UnitHub::Notifications::SESSION_STARTING_SOON + UnitHub::Notifications.fan_out( + scope: UnitHub::Notifications.recipients(session.unit, author_id: session.author_id) + .where(receive_unit_hub_session_reminders: true), + event: event, + dedupe_key: "#{event}:#{session.id}:#{start_at.to_i}", + notifiable: session, + message: UnitHub::Notifications.session_starting_soon_message(session, start_at), + link: UnitHub::Notifications.session_link(session) + ) + end +end diff --git a/app/sidekiq/sync_teams_announcements_job.rb b/app/sidekiq/sync_teams_announcements_job.rb new file mode 100644 index 0000000000..da1f3534c2 --- /dev/null +++ b/app/sidekiq/sync_teams_announcements_job.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class SyncTeamsAnnouncementsJob + include Sidekiq::Job + sidekiq_options queue: :default, retry: false, lock: :until_executed, lock_ttl: 300 + + def perform + UnitHub::Teams::AnnouncementSync.new.call + rescue UnitHub::Teams::Configuration::Error + Rails.logger.warn('Teams announcement sync configuration is invalid; no messages imported.') + end +end diff --git a/app/sidekiq/unit_announcement_notification_job.rb b/app/sidekiq/unit_announcement_notification_job.rb new file mode 100644 index 0000000000..73dbd7676e --- /dev/null +++ b/app/sidekiq/unit_announcement_notification_job.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Tell a unit about an announcement that was published, pinned or meaningfully +# edited. Queued by UnitHub::Notifications.announcement_committed after the save +# commits, with only the announcement id, the event and a version. +class UnitAnnouncementNotificationJob + include Sidekiq::Job + + # How early a scheduled publish may run before it retries instead of giving + # up. Anything further out was rescheduled and has its own job. + EARLY_TOLERANCE = 5.minutes + + # A publish or pin is locked on its version, so a rescheduled publication time + # gets a job of its own. An update is locked on the announcement alone: while + # one update job is waiting or running, a second edit adds nothing, because + # the job reads the announcement as it is when it runs and the debounce would + # hold a second notification back anyway. + sidekiq_options queue: :notifications, + lock: :until_executed, + lock_args_method: lambda { |args| + args[1] == UnitHub::Notifications::ANNOUNCEMENT_UPDATED ? args.first(2) : args + }, + on_conflict: :reject, + retry: 3 + + def perform(announcement_id, event, version) + announcement = UnitAnnouncement.find_by(id: announcement_id) + return if announcement.nil? + + now = Time.current + if event == UnitHub::Notifications::ANNOUNCEMENT_PUBLISHED && version.to_s.start_with?('published-') + return unless announcement.published_at.to_i.to_s == version.delete_prefix('published-') + raise 'Announcement is not published yet, retrying' if announcement.published_at > now && announcement.published_at <= now + EARLY_TOLERANCE + end + return if event == UnitHub::Notifications::ANNOUNCEMENT_PUBLISHED && version.to_s.start_with?('pinned-') && !announcement.pinned + return unless UnitAnnouncement.visible_at(now).exists?(id: announcement.id) + + event == UnitHub::Notifications::ANNOUNCEMENT_UPDATED ? notify_updated(announcement, now) : notify_published(announcement, version) + end + + private + + def notify_published(announcement, version) + UnitHub::Notifications.fan_out( + scope: UnitHub::Notifications.recipients(announcement.unit, author_id: announcement.author_id), + event: UnitHub::Notifications::ANNOUNCEMENT_PUBLISHED, + dedupe_key: "#{UnitHub::Notifications::ANNOUNCEMENT_PUBLISHED}:#{announcement.id}:#{version}", + notifiable: announcement, + message: UnitHub::Notifications.announcement_message(announcement, UnitHub::Notifications::ANNOUNCEMENT_PUBLISHED), + link: UnitHub::Notifications.announcement_link(announcement) + ) + end + + def notify_updated(announcement, now) + event = UnitHub::Notifications::ANNOUNCEMENT_UPDATED + UnitHub::Notifications.fan_out( + scope: UnitHub::Notifications.recipients(announcement.unit, author_id: announcement.author_id), + event: event, + dedupe_key: "#{event}:#{announcement.id}:#{UnitHub::Notifications.announcement_version(announcement)}", + notifiable: announcement, + message: UnitHub::Notifications.announcement_message(announcement, event), + link: UnitHub::Notifications.announcement_link(announcement), + # Anyone told about this announcement within the debounce window, by a + # publish or an earlier update, is left alone this time. + skip: lambda { |user_ids| + Notification.where( + user_id: user_ids, + notifiable: announcement, + event: [UnitHub::Notifications::ANNOUNCEMENT_PUBLISHED, event] + ).where('created_at > ?', now - UnitHub::Notifications::UPDATE_DEBOUNCE).distinct.pluck(:user_id) + } + ) + end +end diff --git a/app/sidekiq/unit_session_notification_job.rb b/app/sidekiq/unit_session_notification_job.rb new file mode 100644 index 0000000000..ca05a7b649 --- /dev/null +++ b/app/sidekiq/unit_session_notification_job.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Tell a unit that a learning session moved, changed where it runs, or was +# cancelled. Queued by UnitHub::Notifications.session_committed after the save +# commits, with the session id, its lock_version and which kind of change. +class UnitSessionNotificationJob + include Sidekiq::Job + + CHANGES = %w[changed cancelled].freeze + + sidekiq_options queue: :notifications, + lock: :until_executed, + on_conflict: :reject, + retry: 3 + + def perform(session_id, version, change) + return unless CHANGES.include?(change) + + session = UnitLearningSession.find_by(id: session_id) + return if session.nil? || !session.published + + # A later save may have changed the answer. A session cancelled since a + # time change hears about the cancellation from its own job, and one put + # back on since a cancellation hears it is back on. + return if (change == 'cancelled') != session.cancelled + + event = UnitHub::Notifications::SESSION_CHANGED + UnitHub::Notifications.fan_out( + scope: UnitHub::Notifications.recipients(session.unit, author_id: session.author_id), + event: event, + dedupe_key: "#{event}:#{session.id}:v#{version}", + notifiable: session, + message: UnitHub::Notifications.session_changed_message(session, change), + link: UnitHub::Notifications.session_link(session) + ) + end +end diff --git a/app/views/notifications_mailer/unit_announcement_published.html.erb b/app/views/notifications_mailer/unit_announcement_published.html.erb new file mode 100644 index 0000000000..7fbb7815eb --- /dev/null +++ b/app/views/notifications_mailer/unit_announcement_published.html.erb @@ -0,0 +1,22 @@ +<% content_for :icon do %>announcement<% end %> +<% announcement = @notification.notifiable %> +<% heading = announcement&.pinned ? 'Pinned announcement' : 'New announcement' %> +<% content_for :preheader do %><%= @notification.message.to_s.truncate(90) %><% end %> +<% content_for :heading do %><%= heading %><% if announcement %> in <%= announcement.unit.code %><% end %><% end %> + +

Hi <%= @user.first_name %>,

+<% if announcement %> +

Your teaching team posted <%= announcement.pinned ? 'a pinned announcement' : 'an announcement' %> on the Unit Hub.

+ <%= render 'notifications_mailer/unit_hub/details', rows: { + 'Unit' => "#{announcement.unit.code} #{announcement.unit.name}", + 'Title' => announcement.title, + 'Summary' => announcement.body.to_s.squish.truncate(280) + } %> +

Open the Unit Hub to read all of it.

+<% else %> + <%= render 'layouts/notification_mail/details', body: @notification.message %> +<% end %> +<% if @notification_url.present? %> + <%= render 'layouts/notification_mail/button', url: @notification_url, label: 'Read the announcement' %> +<% end %> +<%= render 'notifications_mailer/unit_hub/footer' %> diff --git a/app/views/notifications_mailer/unit_announcement_published.text.erb b/app/views/notifications_mailer/unit_announcement_published.text.erb new file mode 100644 index 0000000000..63e4145362 --- /dev/null +++ b/app/views/notifications_mailer/unit_announcement_published.text.erb @@ -0,0 +1,20 @@ +<% announcement = @notification.notifiable -%> +Hi <%= @user.first_name %>, + +<%= @notification.message %> +<% if announcement -%> + +Unit: <%= announcement.unit.code %> <%= announcement.unit.name %> +Title: <%= announcement.title %> +Summary: <%= announcement.body.to_s.squish.truncate(280) %> + +Open the Unit Hub to read all of it. +<% end -%> +<% if @notification_url.present? -%> + +Read the announcement: <%= @notification_url %> +<% end -%> + +You are getting this because Unit Hub updates are on in your notification settings. +You can change them here: +<%= @unsubscribe_url %> diff --git a/app/views/notifications_mailer/unit_announcement_updated.html.erb b/app/views/notifications_mailer/unit_announcement_updated.html.erb new file mode 100644 index 0000000000..3c4cbae39d --- /dev/null +++ b/app/views/notifications_mailer/unit_announcement_updated.html.erb @@ -0,0 +1,21 @@ +<% content_for :icon do %>announcement<% end %> +<% announcement = @notification.notifiable %> +<% content_for :preheader do %><%= @notification.message.to_s.truncate(90) %><% end %> +<% content_for :heading do %>Announcement updated<% if announcement %> in <%= announcement.unit.code %><% end %><% end %> + +

Hi <%= @user.first_name %>,

+<% if announcement %> +

Your teaching team changed an announcement on the Unit Hub.

+ <%= render 'notifications_mailer/unit_hub/details', rows: { + 'Unit' => "#{announcement.unit.code} #{announcement.unit.name}", + 'Title' => announcement.title, + 'Summary' => announcement.body.to_s.squish.truncate(280) + } %> +

Open the Unit Hub to read the current version.

+<% else %> + <%= render 'layouts/notification_mail/details', body: @notification.message %> +<% end %> +<% if @notification_url.present? %> + <%= render 'layouts/notification_mail/button', url: @notification_url, label: 'Read the announcement' %> +<% end %> +<%= render 'notifications_mailer/unit_hub/footer' %> diff --git a/app/views/notifications_mailer/unit_announcement_updated.text.erb b/app/views/notifications_mailer/unit_announcement_updated.text.erb new file mode 100644 index 0000000000..174f6470d8 --- /dev/null +++ b/app/views/notifications_mailer/unit_announcement_updated.text.erb @@ -0,0 +1,20 @@ +<% announcement = @notification.notifiable -%> +Hi <%= @user.first_name %>, + +<%= @notification.message %> +<% if announcement -%> + +Unit: <%= announcement.unit.code %> <%= announcement.unit.name %> +Title: <%= announcement.title %> +Summary: <%= announcement.body.to_s.squish.truncate(280) %> + +Open the Unit Hub to read the current version. +<% end -%> +<% if @notification_url.present? -%> + +Read the announcement: <%= @notification_url %> +<% end -%> + +You are getting this because Unit Hub updates are on in your notification settings. +You can change them here: +<%= @unsubscribe_url %> diff --git a/app/views/notifications_mailer/unit_hub/_details.html.erb b/app/views/notifications_mailer/unit_hub/_details.html.erb new file mode 100644 index 0000000000..39ce823d87 --- /dev/null +++ b/app/views/notifications_mailer/unit_hub/_details.html.erb @@ -0,0 +1,7 @@ +<%# rows: an ordered hash of label => value, all plain text and escaped here. %> +<% body = capture do %> + <% rows.each_with_index do |(label, value), index| %> +

<%= label %>: <%= value %>

+ <% end %> +<% end %> +<%= render 'layouts/notification_mail/details', body: body %> diff --git a/app/views/notifications_mailer/unit_hub/_footer.html.erb b/app/views/notifications_mailer/unit_hub/_footer.html.erb new file mode 100644 index 0000000000..18766cb93f --- /dev/null +++ b/app/views/notifications_mailer/unit_hub/_footer.html.erb @@ -0,0 +1,6 @@ +<% content_for :footer do %> +

+ You are getting this because Unit Hub updates are on in your notification settings. + You can change them in your profile. +

+<% end %> diff --git a/app/views/notifications_mailer/unit_session_changed.html.erb b/app/views/notifications_mailer/unit_session_changed.html.erb new file mode 100644 index 0000000000..64e40b2037 --- /dev/null +++ b/app/views/notifications_mailer/unit_session_changed.html.erb @@ -0,0 +1,21 @@ +<% content_for :icon do %>change<% end %> +<% session = @notification.notifiable %> +<% cancelled = session&.cancelled %> +<% content_for :preheader do %><%= @notification.message.to_s.truncate(90) %><% end %> +<% content_for :heading do %><%= cancelled ? 'Session cancelled' : 'Session changed' %><% if session %> in <%= session.unit.code %><% end %><% end %> + +

Hi <%= @user.first_name %>,

+<% if session && cancelled %> +

<%= @notification.message %>

+

This session will not run.<% if session.join_url.present? %> Its join link has been taken off the Unit Hub.<% end %>

+ <%= render 'notifications_mailer/unit_hub/details', rows: UnitHub::Notifications.session_details(session, at: @notification.created_at) %> +<% elsif session %> +

The time, place or link for a session has changed. Here are the new details.

+ <%= render 'notifications_mailer/unit_hub/details', rows: UnitHub::Notifications.session_details(session, at: Time.current) %> +<% else %> + <%= render 'layouts/notification_mail/details', body: @notification.message %> +<% end %> +<% if @notification_url.present? %> + <%= render 'layouts/notification_mail/button', url: @notification_url, label: 'Open the session' %> +<% end %> +<%= render 'notifications_mailer/unit_hub/footer' %> diff --git a/app/views/notifications_mailer/unit_session_changed.text.erb b/app/views/notifications_mailer/unit_session_changed.text.erb new file mode 100644 index 0000000000..a57e4867a9 --- /dev/null +++ b/app/views/notifications_mailer/unit_session_changed.text.erb @@ -0,0 +1,22 @@ +<% session = @notification.notifiable -%> +Hi <%= @user.first_name %>, + +<%= @notification.message %> +<% if session&.cancelled -%> + +This session will not run.<% if session.join_url.present? %> Its join link has been taken off the Unit Hub.<% end %> +<% end -%> +<% if session -%> + +<% UnitHub::Notifications.session_details(session, at: session.cancelled ? @notification.created_at : Time.current).each do |label, value| -%> +<%= label %>: <%= value %> +<% end -%> +<% end -%> +<% if @notification_url.present? -%> + +Open the session: <%= @notification_url %> +<% end -%> + +You are getting this because Unit Hub updates are on in your notification settings. +You can change them here: +<%= @unsubscribe_url %> diff --git a/app/views/notifications_mailer/unit_session_starting_soon.html.erb b/app/views/notifications_mailer/unit_session_starting_soon.html.erb new file mode 100644 index 0000000000..fea5968a65 --- /dev/null +++ b/app/views/notifications_mailer/unit_session_starting_soon.html.erb @@ -0,0 +1,21 @@ +<% content_for :icon do %>session<% end %> +<% session = @notification.notifiable %> +<% content_for :preheader do %><%= @notification.message.to_s.truncate(90) %><% end %> +<% content_for :heading do %>Session starting soon<% if session %> in <%= session.unit.code %><% end %><% end %> + +

Hi <%= @user.first_name %>,

+<% if session %> +

A session you asked to be reminded about starts in about <%= UnitHub::Notifications::REMINDER_LEAD.in_minutes.to_i %> minutes.

+ <%= render 'notifications_mailer/unit_hub/details', rows: UnitHub::Notifications.session_details(session, at: @notification.created_at) %> +<% else %> + <%= render 'layouts/notification_mail/details', body: @notification.message %> +<% end %> +<% if @notification_url.present? %> + <%= render 'layouts/notification_mail/button', url: @notification_url, label: 'Open the session' %> +<% end %> +<% content_for :footer do %> +

+ You are getting this because session reminders are on in your notification settings. + You can turn them off in your profile. +

+<% end %> diff --git a/app/views/notifications_mailer/unit_session_starting_soon.text.erb b/app/views/notifications_mailer/unit_session_starting_soon.text.erb new file mode 100644 index 0000000000..419fe5927c --- /dev/null +++ b/app/views/notifications_mailer/unit_session_starting_soon.text.erb @@ -0,0 +1,18 @@ +<% session = @notification.notifiable -%> +Hi <%= @user.first_name %>, + +<%= @notification.message %> +<% if session -%> + +<% UnitHub::Notifications.session_details(session, at: @notification.created_at).each do |label, value| -%> +<%= label %>: <%= value %> +<% end -%> +<% end -%> +<% if @notification_url.present? -%> + +Open the session: <%= @notification_url %> +<% end -%> + +You are getting this because session reminders are on in your notification settings. +You can turn them off here: +<%= @unsubscribe_url %> diff --git a/db/migrate/20260914000000_create_unit_hub.rb b/db/migrate/20260914000000_create_unit_hub.rb new file mode 100644 index 0000000000..bf9bf5fa26 --- /dev/null +++ b/db/migrate/20260914000000_create_unit_hub.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class CreateUnitHub < ActiveRecord::Migration[8.0] + def change + create_table :unit_announcements, charset: 'utf8mb4', collation: 'utf8mb4_general_ci' do |t| + t.references :unit, null: false, foreign_key: true + t.references :author, null: true, foreign_key: { to_table: :users, on_delete: :nullify } + t.string :title, null: false, limit: 200 + t.text :body, null: false + t.string :source_url, limit: 2048 + t.boolean :pinned, null: false, default: false + t.datetime :published_at + t.datetime :expires_at + t.timestamps + t.index [:unit_id, :published_at] + end + + create_table :unit_learning_sessions, charset: 'utf8mb4', collation: 'utf8mb4_general_ci' do |t| + t.references :unit, null: false, foreign_key: true + t.references :author, null: true, foreign_key: { to_table: :users, on_delete: :nullify } + t.string :title, null: false, limit: 200 + t.text :description + t.string :kind, null: false, default: 'helphub' + t.datetime :start_at, null: false + t.datetime :end_at, null: false + t.string :timezone, null: false, default: 'Australia/Melbourne' + t.string :location, limit: 300 + t.string :join_url, limit: 2048 + t.string :source_url, limit: 2048 + t.boolean :published, null: false, default: false + t.boolean :cancelled, null: false, default: false + t.string :recurrence, null: false, default: 'none' + t.date :recurrence_until + t.timestamps + t.index [:unit_id, :published, :start_at], name: 'index_unit_sessions_for_feed' + end + + add_column :webcals, :include_learning_sessions, :boolean, null: false, default: false + end +end diff --git a/db/migrate/20260914000001_add_learning_session_lock_version.rb b/db/migrate/20260914000001_add_learning_session_lock_version.rb new file mode 100644 index 0000000000..a9baccc9d3 --- /dev/null +++ b/db/migrate/20260914000001_add_learning_session_lock_version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddLearningSessionLockVersion < ActiveRecord::Migration[8.0] + def change + add_column :unit_learning_sessions, :lock_version, :integer, null: false, default: 0 + end +end diff --git a/db/migrate/20260914000002_add_teams_announcement_sources.rb b/db/migrate/20260914000002_add_teams_announcement_sources.rb new file mode 100644 index 0000000000..1c03aa8c30 --- /dev/null +++ b/db/migrate/20260914000002_add_teams_announcement_sources.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class AddTeamsAnnouncementSources < ActiveRecord::Migration[8.0] + def change + add_column :unit_announcements, :source_provider, :string, null: false, default: 'manual' + add_column :unit_announcements, :external_source_key, :string, limit: 64 + add_column :unit_announcements, :source_mapping_key, :string, limit: 64 + add_column :unit_announcements, :source_channel_key, :string, limit: 64 + add_column :unit_announcements, :external_message_id, :string, limit: 128 + add_column :unit_announcements, :source_updated_at, :datetime + add_column :unit_announcements, :source_imported_at, :datetime + add_column :unit_announcements, :source_checked_at, :datetime + add_index :unit_announcements, [:unit_id, :external_source_key], unique: true, name: 'index_announcements_external_source' + add_index :unit_announcements, [:source_mapping_key, :source_checked_at], name: 'index_announcements_source_scan' + create_table :teams_announcement_sync_states, charset: 'utf8mb4', collation: 'utf8mb4_general_ci' do |t| + t.references :unit, null: false, foreign_key: true + t.string :mapping_key, null: false, limit: 64 + t.string :status, null: false, default: 'pending' + t.datetime :last_attempt_at + t.datetime :last_succeeded_at + t.datetime :next_attempt_at + t.timestamps + t.index :mapping_key, unique: true + end + end +end diff --git a/docs/unit-hub/README.md b/docs/unit-hub/README.md new file mode 100644 index 0000000000..791fecc0d1 --- /dev/null +++ b/docs/unit-hub/README.md @@ -0,0 +1,63 @@ +# Unit hub: announcements, HelpHubs and classes + +The unit hub puts staff-published announcements and scheduled learning sessions in one place. Content belongs to the actual OnTrack unit offering ID, not just a unit code. A student enrolled in SIT111 cannot read SIT102 content by changing a browser filter or an API ID. + +## Delivered behaviour + +- A signed-in feed combines announcements and the next 90 days of sessions from active units in which the user is currently enrolled or assigned as teaching staff. +- Assigned tutors and convenors can create, edit, publish and remove announcements, and create, edit and cancel sessions. Observer-only staff can read published content, but cannot manage it. A global staff or admin account does not automatically gain unrelated unit hub access. +- Draft, future-published and expired announcements do not appear in the student feed. Unpublished sessions do not appear. Feed data and subscription responses use `Cache-Control: private, no-store`. +- HelpHub, lecture, seminar, workshop and other session types support a safe HTTPS join link and optional location/source link. Content is plain text; the client must not interpret it as HTML. The server does not fetch staff links. +- A weekly schedule repeats on the original weekday at the original local time. Staff can create separate Thursday and Friday schedules. Recurrence must end within six months and produces at most 27 occurrences. There is no unbounded rule parser. +- Existing calendar subscriptions can explicitly opt in to learning sessions. This also shares their session titles, location and join links with the chosen calendar provider. Existing unit exclusions apply. Announcements and announcement bodies are never included. + +## API contract + +All routes below are relative to `/api` and require the existing OnTrack authentication header, except the existing token-protected public calendar URL. + +| Route | Result | +| --- | --- | +| `GET /unit_hub` | `{units, announcements, sessions, announcements_truncated, window_start, window_end}` | +| `GET /units/:unit_id/announcements` | Assigned non-observer teaching staff only; all announcement rows, including drafts and expired rows | +| `POST /units/:unit_id/announcements` | Create `{announcement: {...}}`; returns the saved row | +| `PUT /units/:unit_id/announcements/:id` | Update allowed fields through `{announcement: {...}}`; returns the saved row | +| `DELETE /units/:unit_id/announcements/:id` | Remove an announcement | +| `GET /units/:unit_id/sessions` | Assigned non-observer teaching staff only; raw schedules, including drafts/cancelled rows | +| `POST /units/:unit_id/sessions` | Create `{session: {...}}`; returns the saved raw schedule | +| `PUT /units/:unit_id/sessions/:id` | Update through `{session: {...}}`; returns the saved raw schedule | +| `DELETE /units/:unit_id/sessions/:id` | Cancel the schedule, retaining identity for calendar cancellation notices | +| `GET /webcal` | Existing preferences plus `include_learning_sessions` when enabled | +| `PUT /webcal` | Existing wrapper, e.g. `{webcal: {enabled: true, include_learning_sessions: true}}` | + +Unit rows contain `id`, `code`, `name` and `can_manage`. Announcement rows additionally expose `source_provider`, `managed_externally`, `author_name` and `source_imported_at`; private source keys and credentials are never returned. Unit rows also expose `teams_sync` as `configured` or `not_configured` (configuration status, not proof of a live connection). Announcement rows contain `id`, `unit_id`, `unit_code`, `unit_name`, `title`, `body`, `source_url`, `pinned`, `published_at`, `expires_at` and `updated_at`. Set `published_at` to null to save a draft. Up to 100 announcements appear, pinned first then newest; the truncation flag makes this bound explicit. + +Session rows contain `id`, `unit_id`, `unit_code`, `unit_name`, `title`, `description`, `kind`, `start_at`, `end_at`, `timezone`, `location`, `join_url`, `source_url`, `published`, `cancelled`, `recurrence`, `recurrence_until` and `updated_at`. `kind` is one of `helphub`, `lecture`, `seminar`, `workshop`, `other`. `recurrence` is `none` or `weekly`. New schedules default to unpublished. The feed expands schedules into occurrences and additionally includes `occurrence_id` and `original_start_at`. Staff listing and write responses return the original schedule. Feed cancellation rows have `join_url: null`. + +Date-time inputs must use ISO 8601 and include an explicit UTC offset, for example `2026-10-08T17:00:00+11:00`. `timezone` is an IANA identifier such as `Australia/Melbourne`; `recurrence_until` is `YYYY-MM-DD`. Session end must be after start and within 24 hours. Announcement bodies and session descriptions are limited to 20,000 characters; titles to 200. Links must be HTTPS without whitespace or embedded credentials. Invalid model data produces HTTP 400 with a readable `error`; unauthorized unit IDs or records produce 404 and enrolled users without management rights receive 403. + +## Calendar behaviour and privacy + +Learning sessions default to off, including for existing subscriptions. Once enabled, the subscription checks current enrolled projects, active current units and unit exclusions on every request. Sessions are queried independently from task definitions, so units without tasks still work. Withdrawing from a unit removes its sessions from the next fetched calendar. Users can revoke or rotate the existing subscription token in the existing preferences flow. + +Each occurrence has a stable UID based on schedule ID and week index. Rescheduling updates the same UID; Rails' increasing `lock_version` becomes the iCalendar sequence, so rapid edits remain ordered. Deleting a schedule marks it cancelled and retains a `STATUS:CANCELLED` event without a join link. Cancellation records remain in the rolling 30-day past/six-month future subscription window. Calendar providers choose when they refresh, so changes and revocations are not instant and previously downloaded copies cannot be erased by OnTrack. Manually added Google Calendar events and downloaded ICS files are snapshots; use a subscription for ongoing updates. + +Weekly expansion adds local calendar weeks before UTC export to preserve ordinary HelpHub times across daylight saving. A recurrence landing in a missing clock-change hour follows the time-zone library's forward normalization; use daytime session times or review that occurrence if such a schedule is needed. This release does not support per-occurrence exceptions or automatic timetable import. + +## Deployment and demo boundary + +Run the migrations through `20260914000002` before starting the matching web release. They add `unit_announcements`, `unit_learning_sessions`, the calendar preference and the calendar revision field. Existing data is retained, and existing calendar subscriptions keep their prior content until the user opts in. + +The normal feature uses real authenticated API records. The web application's existing explicit demo mode uses synthetic client-side examples and makes no hub write requests. These migrations and normal seeds contain no screenshot material or invented real class links. The Teams screenshots were examples of student needs, not commands, credentials or authorized access to Teams. Staff can enter approved source and meeting links. The optional [university-managed Teams announcement sync](teams-sync.md) imports only configured student-wide channels and approved staff publishers; it defaults to off and requires university application consent. Existing SSO is unchanged, and students do not sign in again. Imported rows are managed in Teams, preserve source ownership, and remain subject to OnTrack enrolment checks. + +## Verification + +Focused tests cover cross-unit isolation, withdrawn enrolments, inactive units, drafts, expiry, future publication, staff/observer permissions, forged IDs, link/date validation, authentication, weekly DST behavior, bounded recurrence, opt-in subscriptions, unit exclusions, units without tasks, final-day inclusion, stable UID updates and cancellation without stale join links. + +Configure the `DF_TEST_DB_*` settings to a dedicated, seeded test database before running these commands. Never point them at a development database holding work: + +```sh +RAILS_ENV=test bundle exec rake db:migrate +RAILS_ENV=test bundle exec ruby -Itest -r./config/environment -e 'ActiveRecord.maintain_test_schema = false; ARGV.each { |path| require File.expand_path(path) }' test/api/unit_hub_api_test.rb test/models/unit_learning_session_test.rb test/models/unit_hub_calendar_test.rb +``` + +Existing calendar API/model tests should run alongside these before release. The runtime evidence and exact counts are recorded with the delivery, rather than assumed here. diff --git a/docs/unit-hub/teams-sync.md b/docs/unit-hub/teams-sync.md new file mode 100644 index 0000000000..77b8b91caa --- /dev/null +++ b/docs/unit-hub/teams-sync.md @@ -0,0 +1,61 @@ +# Optional automatic Teams announcements + +This connection copies approved Teams announcements into Unit Hub. It is off by default. It uses a university-managed Microsoft application in the background; students continue using the existing OnTrack sign-in. The connection does not change SAML/Entra sign-in, create student accounts or infer enrolment from Teams names. + +OnTrack enrolment decides which unit updates a student receives. Microsoft application consent separately allows the worker to read approved Teams data. Signing in through Microsoft does not grant that permission by itself. [Microsoft permissions and consent](https://learn.microsoft.com/en-us/entra/identity-platform/permissions-consent-overview) + +## University setup + +1. Apply the Unit Hub migrations through `20260914000002` and deploy the matching API and worker code. This adds source metadata and sync checkpoints without changing manual announcements. +2. Have the university administrator approve the data use, exact unit offerings, source channels and staff publishers. Map only channels whose content every student enrolled in that unit offering is entitled to see. Never map a private staff channel or a restricted student subgroup. +3. Register a university-managed, single-tenant Microsoft application. Prefer `ChannelMessage.Read.Group` through the university-approved Teams app installation and resource-specific consent process. This permission is supported for listing and retrieving channel messages. Resource-specific consent limits the grant to approved Teams resources; the application mapping narrows it to particular channels. Broader `ChannelMessage.Read.All` requires a deliberate university decision and appropriate consent. [List permissions](https://learn.microsoft.com/en-us/graph/api/channel-list-messages?view=graph-rest-1.0), [Get-message permissions](https://learn.microsoft.com/en-us/graph/api/chatmessage-get?view=graph-rest-1.0), [Teams resource-specific consent](https://learn.microsoft.com/en-us/microsoftteams/platform/graph-api/rsc/resource-specific-consent) +4. Store the application client secret through the deployment's protected secret process. The worker uses the tenant-specific client credentials flow with `https://graph.microsoft.com/.default`. No student token or extra personal account-linking step is used. This implementation supports Microsoft's public-cloud endpoints only. [Client credentials flow](https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow) +5. Configure the values below. Use the actual OnTrack **unit offering database ID**, a verified team/channel pair, and the Entra object IDs of approved staff publishers. A unit code, email address or display name is not an identity mapping. + +```dotenv +DF_TEAMS_ANNOUNCEMENTS_ENABLED=true +DF_TEAMS_TENANT_ID= +DF_TEAMS_CLIENT_ID= +DF_TEAMS_CLIENT_SECRET= +DF_TEAMS_CHANNEL_MAPPINGS=[{"unit_id":123,"team_id":"","channel_id":"19:@thread.tacv2","publisher_ids":[""],"student_visible":true}] +``` + +Replace every placeholder. Keep the mapping JSON on one line without surrounding quotes when using the supplied production validator. Disabled defaults are `DF_TEAMS_ANNOUNCEMENTS_ENABLED=false` and `DF_TEAMS_CHANNEL_MAPPINGS=[]`. + +The API and main worker both need the enabled flag, tenant ID and mapping JSON. **Only the main worker receives the client ID and secret.** Do not pass application credentials to the web build, student browser, PDF worker, migration service or fictional demo. The deployment repository provides the service isolation and validation rules. + +Between one and twenty mappings are allowed when enabled. Every mapping requires `student_visible:true` and one to one hundred approved publisher IDs. Invalid identifiers, duplicate exact mappings and missing approval fields are rejected. The visibility declaration is an operator's explicit assertion; the importer does not verify that the Teams membership list equals OnTrack enrolment. University review of this mapping remains necessary. + +## Operation and boundaries + +`SyncTeamsAnnouncementsJob` is scheduled every five minutes. To request a run immediately, execute this inside the configured main worker: + +```sh +bundle exec rake teams:sync_announcements +``` + +The command reports status and counts without printing credentials or message contents. The Unit Hub `teams_sync: configured` label means a valid mapping exists. It does not prove consent succeeded or that the most recent import completed. + +Each run reads at most two pages of fifty recent root messages per mapping, then revisits up to twenty-five older imported messages, starting with those checked longest ago. The listing API excludes replies by default; the importer also rejects replies, system/application messages and posts from anyone outside the approved publisher list. Returned team/channel identity and individual lookup IDs must match the requested source. This does not copy general student discussion. Older messages that were never imported are outside this bounded initial import. [Microsoft channel listing behaviour](https://learn.microsoft.com/en-us/graph/api/channel-list-messages?view=graph-rest-1.0) + +Graph requests use fixed HTTPS Microsoft hosts and do not follow redirects. Pagination links must remain on the exact configured channel messages path. Each response is capped at 2 MiB; the client has a 180-second request budget checked before requests and during body reads, with five-second connection/write and ten-second read timeouts. A blocked transport operation remains subject to its own timeout. + +HTTP 429 triggers a persisted cooldown across mappings and worker runs. Numeric `Retry-After` values are bounded to 60–86,400 seconds; absent or unsupported values use 300 seconds. No immediate retry loop is used. + +## Copies, changes and removal + +Imported posts have stable source IDs and are marked `microsoft_teams` / externally managed. Repeated imports update the same record. Source identities are separate from manual announcements, and the normal staff API refuses to edit or delete imported copies. Staff make those changes in Teams. + +The importer converts HTML to plain text. It does not fetch attachments, execute markup or load external images. Source links must match the configured tenant, team, channel and message. Stored links are rebuilt as a Teams message link with only the verified team and tenant parameters. The serializer does not expose source keys or application credentials. + +A post missing from a limited recent listing is **not** assumed deleted. Older imported records are checked individually. An explicit deleted message or individual 404/410 unpublishes that copy. A missing source channel, HTTP 401/403, or a token-endpoint HTTP 400 unpublishes affected mapped copies. Temporary network/server failures retain previously verified copies only until their seven-day expiry. Older provider versions do not overwrite newer stored text. + +Disabling the feature, removing a mapping or changing its tenant, unit, channel or approved publishers changes the visibility rules immediately on subsequent API requests. Apply metadata changes to the API and worker together. Both student and staff lists enforce these rules. An unchanged source channel has a separate stable identity, allowing old posts to be rechecked under a changed publisher policy before becoming visible again. A new source channel starts its own bounded import. + +Operators can inspect `TeamsAnnouncementSyncState` for `status`, `last_attempt_at`, `last_succeeded_at` and `next_attempt_at`. Status values are `pending`, `synced`, `failed` and `throttled`. Check these in an authorised operational session; do not publish message bodies, tokens or secrets in issue comments. Sync checkpoints and imported rows are removed with their unit. + +## Verification and remaining scope + +Automated verification uses mocked Microsoft responses and an isolated seeded test database. No real university application credentials or tenant connection were available during implementation. Before enabling a university deployment, verify an approved staff post appears for the correct enrolled student, then test an edit, deletion, an unapproved publisher, another unit's student and a disabled mapping. Confirm the deployed worker and API share the same tenant/mapping metadata. + +HelpHub and class times remain structured sessions maintained by assigned OnTrack teaching staff. This importer does not infer timetable dates or meeting schedules from free-form posts. Automatic timetable integration needs an approved authoritative source and separate implementation. Existing calendar subscription consent, unit exclusions and the distinction between one-off copies and subscriptions are unchanged. diff --git a/lib/tasks/teams_announcements.rake b/lib/tasks/teams_announcements.rake new file mode 100644 index 0000000000..ff2cb94173 --- /dev/null +++ b/lib/tasks/teams_announcements.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :teams do + desc 'Sync configured student-visible Teams announcements using application credentials' + task sync_announcements: :environment do + result = UnitHub::Teams::AnnouncementSync.new.call + puts "Teams announcement sync: #{result[:status]}; synced mappings: #{result[:synced]}; failed mappings: #{result[:failed]}." + rescue UnitHub::Teams::Configuration::Error + abort 'Teams announcement sync configuration is invalid. Check the operator setup guide.' + end +end diff --git a/test/api/unit_hub_api_test.rb b/test/api/unit_hub_api_test.rb new file mode 100644 index 0000000000..58bdd2b15a --- /dev/null +++ b/test/api/unit_hub_api_test.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +require 'test_helper' + +class UnitHubApiTest < ActiveSupport::TestCase + include Rack::Test::Methods + include TestHelpers::AuthHelper + + def app + Rails.application + end + + setup do + @student = FactoryBot.create(:user, :student) + @unit = FactoryBot.create(:unit, with_students: false, task_count: 0, code: 'SIT111') + @other_unit = FactoryBot.create(:unit, with_students: false, task_count: 0, code: 'SIT102') + @project = @unit.enrol_student(@student, @unit.tutorials.first.campus) + @staff = @unit.unit_roles.where(role: Role.convenor).first.user + @announcement = @unit.unit_announcements.create!(title: 'Study update', body: 'Bring your questions.', published_at: 1.hour.ago) + @session = @unit.unit_learning_sessions.create!(title: 'HelpHub', start_at: 1.day.from_now, end_at: 1.day.from_now + 1.hour, + published: true, join_url: 'https://teams.microsoft.com/l/meetup-join/example') + end + + def test_feed_contains_only_enrolled_unit_published_unexpired_content + @other_unit.unit_announcements.create!(title: 'Other unit secret', body: 'Other students only.', published_at: 1.hour.ago) + @other_unit.unit_learning_sessions.create!(title: 'Other class', start_at: 1.day.from_now, end_at: 1.day.from_now + 1.hour, published: true) + @unit.unit_announcements.create!(title: 'Draft', body: 'Draft body') + @unit.unit_announcements.create!(title: 'Expired', body: 'Old body', published_at: 2.days.ago, expires_at: 1.day.ago) + @unit.unit_announcements.create!(title: 'Scheduled', body: 'Future body', published_at: 1.day.from_now) + @unit.unit_learning_sessions.create!(title: 'Draft class', start_at: 1.day.from_now, end_at: 1.day.from_now + 1.hour) + add_auth_header_for(user: @student) + get '/api/unit_hub' + + assert_equal 200, last_response.status, last_response.body + body = JSON.parse(last_response.body) + assert_equal [@unit.id], body['units'].map { |unit| unit['id'] } + assert_equal [false], body['units'].map { |unit| unit['can_manage'] } + assert_equal [@announcement.id], body['announcements'].map { |row| row['id'] } + assert_equal [@session.id], body['sessions'].map { |row| row['id'] } + assert_equal 'private, no-store', last_response.headers['cache-control'] + end + + def test_withdrawal_and_inactive_units_are_removed_on_next_request + add_auth_header_for(user: @student) + @project.update!(enrolled: false) + get '/api/unit_hub' + assert_equal [], JSON.parse(last_response.body)['units'] + assert_equal [], JSON.parse(last_response.body)['announcements'] + assert_equal [], JSON.parse(last_response.body)['sessions'] + + @project.update!(enrolled: true) + @unit.update!(active: false) + get '/api/unit_hub' + assert_equal [], JSON.parse(last_response.body)['units'] + end + + def test_global_staff_role_does_not_grant_unassigned_unit_access + add_auth_header_for(user: FactoryBot.create(:user, :convenor)) + get '/api/unit_hub' + assert_equal [], JSON.parse(last_response.body)['units'] + get "/api/units/#{@unit.id}/announcements" + assert_equal 404, last_response.status + end + + def test_student_cannot_manage_even_their_own_unit + add_auth_header_for(user: @student) + get "/api/units/#{@unit.id}/announcements" + assert_equal 403, last_response.status, last_response.body + assert_no_difference 'UnitAnnouncement.count' do + post "/api/units/#{@unit.id}/announcements", announcement: { title: 'Injected', body: 'No' } + end + assert_equal 403, last_response.status + put "/api/units/#{@unit.id}/sessions/#{@session.id}", session: { title: 'Changed' } + assert_equal 403, last_response.status + delete "/api/units/#{@unit.id}/announcements/#{@announcement.id}" + assert_equal 403, last_response.status + end + + def test_assigned_staff_can_create_edit_list_drafts_and_remove_content + add_auth_header_for(user: @staff) + post "/api/units/#{@unit.id}/announcements", announcement: { title: 'New draft', body: 'Staff note', unit_id: @other_unit.id } + assert_equal 201, last_response.status, last_response.body + id = JSON.parse(last_response.body)['id'] + assert_equal @unit.id, UnitAnnouncement.find(id).unit_id + assert_equal @staff.id, UnitAnnouncement.find(id).author_id + + put "/api/units/#{@unit.id}/announcements/#{id}", announcement: { published_at: Time.current.iso8601 } + assert_equal 200, last_response.status, last_response.body + get "/api/units/#{@unit.id}/announcements" + assert_includes JSON.parse(last_response.body).map { |row| row['id'] }, id + delete "/api/units/#{@unit.id}/announcements/#{id}" + assert_equal 200, last_response.status + assert_not UnitAnnouncement.exists?(id) + end + + def test_staff_can_create_weekly_draft_then_publish_and_edit_a_session + add_auth_header_for(user: @staff) + start_at = 2.days.from_now.in_time_zone('Australia/Melbourne').change(hour: 17, min: 0, sec: 0) + post "/api/units/#{@unit.id}/sessions", session: { + title: 'Weekly lecture', kind: 'lecture', start_at: start_at.iso8601, + end_at: (start_at + 1.hour).iso8601, timezone: 'Australia/Melbourne', + recurrence: 'weekly', recurrence_until: (start_at.to_date + 2.weeks).iso8601, + join_url: 'https://teams.microsoft.com/l/meetup-join/example' + } + assert_equal 201, last_response.status, last_response.body + row = JSON.parse(last_response.body) + assert_equal false, row['published'] + id = row['id'] + get "/api/units/#{@unit.id}/sessions" + assert_includes JSON.parse(last_response.body).map { |record| record['id'] }, id + put "/api/units/#{@unit.id}/sessions/#{id}", session: { published: true, title: 'Lecture and questions' } + assert_equal 200, last_response.status, last_response.body + add_auth_header_for(user: @student) + get '/api/unit_hub' + occurrences = JSON.parse(last_response.body)['sessions'].select { |record| record['id'] == id } + assert_equal 3, occurrences.length + assert_equal ['Lecture and questions'], occurrences.map { |record| record['title'] }.uniq + end + + def test_cross_unit_record_ids_cannot_be_modified_or_deleted + add_auth_header_for(user: @staff) + other = @other_unit.unit_announcements.create!(title: 'Private', body: 'Protected') + put "/api/units/#{@unit.id}/announcements/#{other.id}", announcement: { title: 'Changed' } + assert_equal 404, last_response.status + delete "/api/units/#{@unit.id}/announcements/#{other.id}" + assert_equal 404, last_response.status + assert_equal 'Private', other.reload.title + post "/api/units/#{@other_unit.id}/sessions", session: { title: 'No' } + assert_equal 404, last_response.status + end + + def test_observer_staff_cannot_publish + observer = FactoryBot.create(:user, :tutor) + @unit.unit_roles.create!(user: observer, role: Role.tutor, observer_only: true) + add_auth_header_for(user: observer) + get '/api/unit_hub' + assert_equal false, JSON.parse(last_response.body)['units'].first['can_manage'] + post "/api/units/#{@unit.id}/announcements", announcement: { title: 'No', body: 'No' } + assert_equal 403, last_response.status + end + + def test_session_cancellation_retains_identity_but_hides_join_link + add_auth_header_for(user: @staff) + delete "/api/units/#{@unit.id}/sessions/#{@session.id}" + assert_equal 200, last_response.status + assert @session.reload.cancelled? + add_auth_header_for(user: @student) + get '/api/unit_hub' + row = JSON.parse(last_response.body)['sessions'].first + assert row['cancelled'] + assert_nil row['join_url'] + assert_equal "#{@session.id}-0", row['occurrence_id'] + end + + def test_invalid_links_offsets_and_schedules_are_rejected + add_auth_header_for(user: @staff) + %w[javascript:alert(1) http://example.com https://user:password@example.com].each do |url| + put "/api/units/#{@unit.id}/sessions/#{@session.id}", session: { join_url: url } + assert_equal 400, last_response.status, last_response.body + end + put "/api/units/#{@unit.id}/sessions/#{@session.id}", session: { start_at: '2026-10-01T17:00:00' } + assert_equal 400, last_response.status + put "/api/units/#{@unit.id}/sessions/#{@session.id}", session: { start_at: '2026-10-01T17:00:00', end_at: '2026-10-01T18:00:00' } + assert_equal 400, last_response.status, last_response.body + put "/api/units/#{@unit.id}/announcements/#{@announcement.id}", announcement: { published_at: 'not a date' } + assert_equal 400, last_response.status, last_response.body + put "/api/units/#{@unit.id}/sessions/#{@session.id}", session: { timezone: 'Bad/Zone' } + assert_equal 400, last_response.status + put "/api/units/#{@unit.id}/sessions/#{@session.id}", session: { end_at: 1.day.ago.iso8601 } + assert_equal 400, last_response.status + end + + def test_calendar_learning_sessions_preference_is_saved_only_for_current_user + other = FactoryBot.create(:user, :student) + other_cal = Webcal.create!(user: other, guid: SecureRandom.uuid) + add_auth_header_for(user: @student) + put '/api/webcal', webcal: { enabled: true, include_learning_sessions: true } + assert_equal 200, last_response.status, last_response.body + assert_equal true, JSON.parse(last_response.body)['include_learning_sessions'] + get '/api/webcal' + assert_equal true, JSON.parse(last_response.body)['include_learning_sessions'] + assert_not other_cal.reload.include_learning_sessions? + put '/api/webcal', webcal: { include_learning_sessions: false } + assert_equal false, JSON.parse(last_response.body)['include_learning_sessions'] + end + + def test_authentication_required_for_feed_and_writes + get '/api/unit_hub' + assert_equal 419, last_response.status + post "/api/units/#{@unit.id}/announcements", announcement: { title: 'No', body: 'No' } + assert_equal 419, last_response.status + end +end diff --git a/test/models/unit_learning_session_test.rb b/test/models/unit_learning_session_test.rb new file mode 100644 index 0000000000..cc97258984 --- /dev/null +++ b/test/models/unit_learning_session_test.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'test_helper' + +class UnitLearningSessionTest < ActiveSupport::TestCase + setup do + @unit = FactoryBot.create(:unit, with_students: false, task_count: 0) + end + + def test_weekly_occurrences_keep_local_time_across_daylight_saving + session = @unit.unit_learning_sessions.create!(title: 'Thursday HelpHub', timezone: 'Australia/Melbourne', + start_at: Time.iso8601('2026-10-01T17:00:00+10:00'), end_at: Time.iso8601('2026-10-01T18:00:00+10:00'), + recurrence: 'weekly', recurrence_until: '2026-10-15') + occurrences = session.occurrences(from: Time.iso8601('2026-09-01T00:00:00Z'), to: Time.iso8601('2026-11-01T00:00:00Z')) + assert_equal 3, occurrences.length + assert_equal [17, 17, 17], occurrences.map { |row| row[:start_at].hour } + assert_equal [10.hours, 11.hours, 11.hours], occurrences.map { |row| row[:start_at].utc_offset } + assert_equal [7, 6, 6], occurrences.map { |row| row[:start_at].utc.hour } + assert_equal 3, occurrences.pluck(:occurrence_id).uniq.length + end + + def test_weekly_recurrence_requires_a_bounded_end + session = @unit.unit_learning_sessions.new(title: 'HelpHub', start_at: 1.day.from_now, end_at: 1.day.from_now + 1.hour, recurrence: 'weekly') + assert_not session.valid? + session.recurrence_until = 1.year.from_now.to_date + assert_not session.valid? + session.recurrence_until = 1.month.from_now.to_date + assert session.valid?, session.errors.full_messages.join(', ') + end + + def test_link_and_content_limits + session = @unit.unit_learning_sessions.new(title: 'HelpHub', start_at: 1.day.from_now, end_at: 1.day.from_now + 1.hour) + ["https://example.com\nBEGIN:VEVENT", '//example.com', 'data:text/html,test', 'https://a:b@example.com'].each do |url| + session.join_url = url + assert_not session.valid?, url + end + session.join_url = 'https://teams.microsoft.com/l/meetup-join/example?context=%7B%7D' + assert session.valid?, session.errors.full_messages.join(', ') + session.description = 'a' * 20_001 + assert_not session.valid? + end +end diff --git a/test/services/teams_announcement_sync_test.rb b/test/services/teams_announcement_sync_test.rb new file mode 100644 index 0000000000..e8a6cd0c9e --- /dev/null +++ b/test/services/teams_announcement_sync_test.rb @@ -0,0 +1,265 @@ +# frozen_string_literal: true + +require 'test_helper' + +class TeamsAnnouncementSyncTest < ActiveSupport::TestCase + include Rack::Test::Methods + include TestHelpers::AuthHelper + + TENANT = '11111111-1111-4111-8111-111111111111' + CLIENT = '22222222-2222-4222-8222-222222222222' + TEAM = '33333333-3333-4333-8333-333333333333' + PUBLISHER = '44444444-4444-4444-8444-444444444444' + CHANNEL = '19:approved-channel@thread.tacv2' + + def app + Rails.application + end + + setup do + @unit = FactoryBot.create(:unit, with_students: false, task_count: 0) + @student = FactoryBot.create(:user, :student) + @project = @unit.enrol_student(@student, @unit.tutorials.first.campus) + @now = Time.current.change(usec: 0) + @mapping_hash = { unit_id: @unit.id, team_id: TEAM, channel_id: CHANNEL, publisher_ids: [PUBLISHER], student_visible: true } + @env = { 'DF_TEAMS_ANNOUNCEMENTS_ENABLED' => 'true', 'DF_TEAMS_TENANT_ID' => TENANT, + 'DF_TEAMS_CLIENT_ID' => CLIENT, 'DF_TEAMS_CLIENT_SECRET' => 'unit-test-secret', + 'DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash].to_json } + @path = "/v1.0/teams/#{TEAM}/channels/#{ERB::Util.url_encode(CHANNEL)}/messages" + @list_url = "https://graph.microsoft.com#{@path}?$top=50" + stub_request(:post, "https://login.microsoftonline.com/#{TENANT}/oauth2/v2.0/token") + .to_return(body: { access_token: 'test-access-token', token_type: 'Bearer' }.to_json) + end + + def teams_message(id = '12345', **changes) + { 'id' => id, 'messageType' => 'message', 'replyToId' => nil, + 'from' => { 'user' => { 'id' => PUBLISHER } }, + 'channelIdentity' => { 'teamId' => TEAM, 'channelId' => CHANNEL }, + 'subject' => 'Weekly update', 'createdDateTime' => (@now - 1.hour).iso8601, + 'lastModifiedDateTime' => (@now - 1.hour).iso8601, + 'body' => { 'contentType' => 'html', 'content' => '

Bring & discuss questions.

' }, + 'webUrl' => "https://teams.microsoft.com/l/message/#{ERB::Util.url_encode(CHANNEL)}/#{id}?groupId=#{TEAM}&tenantId=#{TENANT}" }.merge(changes.transform_keys(&:to_s)) + end + + def feed(*messages) + stub_request(:get, @list_url).to_return(body: { value: messages }.to_json) + end + + def sync(now: @now, env: @env) + UnitHub::Teams::AnnouncementSync.new(configuration: UnitHub::Teams::Configuration.new(env), now: now).call + end + + def with_env(values = @env) + previous = values.keys.to_h { |key| [key, ENV[key]] } + values.each { |key, value| value.nil? ? ENV.delete(key) : ENV[key] = value } + yield + ensure + previous.each { |key, value| value.nil? ? ENV.delete(key) : ENV[key] = value } + end + + def test_import_is_plain_text_idempotent_and_owned_by_its_source + manual = @unit.unit_announcements.create!(title: 'Manual', body: 'Keep this', published_at: @now) + feed(teams_message) + assert_equal 1, sync[:synced] + record = @unit.unit_announcements.where(source_provider: 'microsoft_teams').sole + assert_equal 'Weekly update', record.title + assert_includes record.body, 'Bring & discuss questions.' + assert_not_includes record.body, '<' + assert_nil record.author_id + assert_equal @now, record.source_imported_at + assert_equal @now + 7.days, record.expires_at + assert_no_difference('UnitAnnouncement.count') { sync } + assert_equal 'Keep this', manual.reload.body + end + + def test_edits_update_the_same_record_and_source_deletion_hides_it + feed(teams_message) + sync + record = @unit.unit_announcements.sole + feed(teams_message(subject: 'Changed notice', lastModifiedDateTime: @now.iso8601)) + sync(now: @now + 5.minutes) + assert_equal 'Changed notice', record.reload.title + assert_equal 1, @unit.unit_announcements.count + feed(teams_message(deletedDateTime: @now.iso8601, from: nil)) + sync(now: @now + 10.minutes) + assert_nil record.reload.published_at + end + + def test_student_system_reply_application_and_wrong_channel_posts_are_not_imported + feed(teams_message('1', from: { 'user' => { 'id' => CLIENT } }), + teams_message('2', messageType: 'systemEventMessage'), teams_message('3', replyToId: '1'), + teams_message('4', from: { 'application' => { 'id' => CLIENT } }), + teams_message('5', channelIdentity: { 'teamId' => CLIENT, 'channelId' => CHANNEL })) + assert_no_difference('UnitAnnouncement.count') { sync } + end + + def test_source_link_must_match_the_mapped_tenant_team_channel_and_message + feed(teams_message(webUrl: "https://teams.microsoft.com/l/message/#{ERB::Util.url_encode(CHANNEL)}/other?groupId=#{TEAM}&tenantId=#{TENANT}")) + assert_no_difference('UnitAnnouncement.count') { sync } + feed(teams_message(webUrl: 'https://evil.example/private')) + assert_no_difference('UnitAnnouncement.count') { sync } + end + + def test_old_message_omitted_from_recent_pages_is_rechecked_not_assumed_deleted + feed(teams_message) + sync + record = @unit.unit_announcements.sole + feed + lookup = stub_request(:get, "https://graph.microsoft.com#{@path}/12345").to_return(body: teams_message(subject: 'Old edited notice', lastModifiedDateTime: @now.iso8601).to_json) + sync(now: @now + 5.minutes) + assert_requested lookup + assert_equal 'Old edited notice', record.reload.title + assert_not_nil record.published_at + stub_request(:get, "https://graph.microsoft.com#{@path}/12345").to_return(status: 404) + sync(now: @now + 10.minutes) + assert_nil record.reload.published_at + end + + def test_old_message_lookup_cannot_import_a_different_response_id + feed(teams_message) + sync + feed + stub_request(:get, "https://graph.microsoft.com#{@path}/12345").to_return(body: teams_message('999').to_json) + assert_no_difference('UnitAnnouncement.count') { assert_equal 1, sync(now: @now + 5.minutes)[:failed] } + assert_equal 'failed', TeamsAnnouncementSyncState.sole.status + end + + def test_revoked_graph_access_unpublishes_snapshots + feed(teams_message) + sync + stub_request(:get, @list_url).to_return(status: 403, body: 'private-provider-body') + assert_equal 1, sync(now: @now + 5.minutes)[:failed] + assert_nil @unit.unit_announcements.sole.published_at + end + + def test_deleted_source_channel_and_revoked_client_credentials_hide_snapshots + feed(teams_message) + sync + stub_request(:get, @list_url).to_return(status: 404) + assert_equal 1, sync(now: @now + 5.minutes)[:failed] + assert_nil @unit.unit_announcements.sole.published_at + feed(teams_message) + sync(now: @now + 10.minutes) + stub_request(:post, "https://login.microsoftonline.com/#{TENANT}/oauth2/v2.0/token") + .to_return(status: 400, body: { error: 'invalid_client', error_description: 'private-provider-body' }.to_json) + assert_equal 1, sync(now: @now + 15.minutes)[:failed] + assert_nil @unit.unit_announcements.sole.published_at + end + + def test_historical_scan_is_bounded_and_rotates_oldest_rows + feed(teams_message) + sync + template = @unit.unit_announcements.sole + 26.times do |index| + record = template.dup + record.external_message_id = (5000 + index).to_s + record.external_source_key = Digest::SHA256.hexdigest(JSON.generate([TENANT, TEAM, CHANNEL, record.external_message_id])) + record.save! + end + feed + lookup = stub_request(:get, %r{https://graph.microsoft.com#{Regexp.escape(URI::DEFAULT_PARSER.unescape(@path))}/[0-9]+$}).to_return do |request| + { body: teams_message(request.uri.path.split('/').last).to_json } + end + sync(now: @now + 5.minutes) + assert_requested lookup, times: 25 + assert_equal 2, @unit.unit_announcements.where(source_checked_at: @now).count + sync(now: @now + 10.minutes) + assert_equal 0, @unit.unit_announcements.where(source_checked_at: @now).count + end + + def test_manual_row_with_colliding_external_key_is_never_overwritten + key = Digest::SHA256.hexdigest(JSON.generate([TENANT, TEAM, CHANNEL, '12345'])) + manual = @unit.unit_announcements.create!(title: 'Manual content', body: 'Keep this', external_source_key: key) + feed(teams_message) + sync + assert_equal 'Manual content', manual.reload.title + assert_equal 'manual', manual.source_provider + assert_equal 1, @unit.unit_announcements.count + end + + def test_throttle_cooldown_is_persisted_and_prevents_early_retry + request = stub_request(:get, @list_url).to_return(status: 429, headers: { 'Retry-After' => '1800' }) + assert_equal 'throttled', sync[:status] + assert_equal @now + 30.minutes, TeamsAnnouncementSyncState.sole.next_attempt_at + sync(now: @now + 5.minutes) + assert_requested request, times: 1 + end + + def test_disabling_connection_removing_publishers_or_changing_tenant_hides_copies_immediately + feed(teams_message) + sync + with_env { assert_equal 1, UnitAnnouncement.visible_at(@now).where(unit: @unit).count } + [@env.merge('DF_TEAMS_ANNOUNCEMENTS_ENABLED' => 'false'), + @env.merge('DF_TEAMS_TENANT_ID' => CLIENT), + @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash.merge(publisher_ids: [CLIENT])].to_json)].each do |values| + with_env(values) { assert_empty UnitAnnouncement.visible_at(@now).where(unit: @unit) } + end + end + + def test_still_approved_old_posts_are_revalidated_after_publisher_configuration_changes + feed(teams_message) + sync + old_key = @unit.unit_announcements.sole.source_mapping_key + feed + stub_request(:get, "https://graph.microsoft.com#{@path}/12345").to_return(body: teams_message.to_json) + changed = @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash.merge(publisher_ids: [PUBLISHER, CLIENT])].to_json) + sync(now: @now + 5.minutes, env: changed) + assert_not_equal old_key, @unit.unit_announcements.sole.source_mapping_key + with_env(changed) { assert_equal 1, UnitAnnouncement.visible_at(@now + 5.minutes).where(unit: @unit).count } + end + + def test_transient_failure_keeps_only_a_time_limited_snapshot + feed(teams_message) + sync + stub_request(:get, @list_url).to_timeout + sync(now: @now + 1.day) + with_env do + assert_equal 1, UnitAnnouncement.visible_at(@now + 1.day).where(unit: @unit).count + assert_empty UnitAnnouncement.visible_at(@now + 8.days).where(unit: @unit) + end + end + + def test_source_metadata_does_not_bypass_enrolment_or_expose_integration_credentials + feed(teams_message) + sync + with_env do + add_auth_header_for(user: @student) + get '/api/unit_hub' + assert_equal 200, last_response.status, last_response.body + body = JSON.parse(last_response.body) + assert_equal 'configured', body['units'].first['teams_sync'] + row = body['announcements'].first + assert_equal true, row['managed_externally'] + assert_equal 'Teaching team', row['author_name'] + %w[external_source_key source_mapping_key source_channel_key external_message_id].each { |key| assert_not row.key?(key) } + assert_not_includes last_response.body, 'unit-test-secret' + @project.update!(enrolled: false) + get '/api/unit_hub' + assert_empty JSON.parse(last_response.body)['announcements'] + end + end + + def test_staff_cannot_overwrite_or_delete_imported_posts_and_disabled_sources_are_hidden_from_staff_list + feed(teams_message) + sync + record = @unit.unit_announcements.sole + add_auth_header_for(user: @unit.unit_roles.where(role: Role.convenor).first.user) + put "/api/units/#{@unit.id}/announcements/#{record.id}", announcement: { title: 'Override' } + assert_equal 403, last_response.status + delete "/api/units/#{@unit.id}/announcements/#{record.id}" + assert_equal 403, last_response.status + with_env(@env.merge('DF_TEAMS_ANNOUNCEMENTS_ENABLED' => 'false')) do + get "/api/units/#{@unit.id}/announcements" + assert_empty JSON.parse(last_response.body) + end + assert_equal 'Weekly update', record.reload.title + end + + def test_unit_cleanup_removes_sync_state_and_source_rows + feed(teams_message) + sync + @unit.destroy! + assert_not TeamsAnnouncementSyncState.exists?(unit_id: @unit.id) + assert_not UnitAnnouncement.exists?(unit_id: @unit.id) + end +end diff --git a/test/services/teams_graph_client_test.rb b/test/services/teams_graph_client_test.rb new file mode 100644 index 0000000000..47bfd373df --- /dev/null +++ b/test/services/teams_graph_client_test.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'test_helper' + +class TeamsGraphClientTest < ActiveSupport::TestCase + TENANT = '11111111-1111-4111-8111-111111111111' + CLIENT = '22222222-2222-4222-8222-222222222222' + TEAM = '33333333-3333-4333-8333-333333333333' + PUBLISHER = '44444444-4444-4444-8444-444444444444' + CHANNEL = '19:approved-channel@thread.tacv2' + TOKEN_URL = "https://login.microsoftonline.com/#{TENANT}/oauth2/v2.0/token" + PATH = "/v1.0/teams/#{TEAM}/channels/#{ERB::Util.url_encode(CHANNEL)}/messages" + LIST_URL = "https://graph.microsoft.com#{PATH}?$top=50" + + setup do + @mapping_hash = { unit_id: 123, team_id: TEAM, channel_id: CHANNEL, publisher_ids: [PUBLISHER], student_visible: true } + @env = { 'DF_TEAMS_ANNOUNCEMENTS_ENABLED' => 'true', 'DF_TEAMS_TENANT_ID' => TENANT, + 'DF_TEAMS_CLIENT_ID' => CLIENT, 'DF_TEAMS_CLIENT_SECRET' => 'unit-test-secret', + 'DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash].to_json } + @config = UnitHub::Teams::Configuration.new(@env) + @mapping = @config.mappings.first + @client = UnitHub::Teams::GraphClient.new(@config.credentials) + @token_stub = stub_request(:post, TOKEN_URL).with(body: { + client_id: CLIENT, client_secret: 'unit-test-secret', grant_type: 'client_credentials', scope: 'https://graph.microsoft.com/.default' + }).to_return(status: 200, body: { access_token: 'test-access-token', token_type: 'Bearer' }.to_json) + end + + def test_default_off_needs_no_credentials_or_network + config = UnitHub::Teams::Configuration.new({}) + assert_empty config.mappings + assert_empty config.visible_mapping_keys + assert_equal 'disabled', UnitHub::Teams::AnnouncementSync.new(configuration: config).call[:status] + assert_not_requested @token_stub + end + + def test_visibility_configuration_does_not_require_client_secret + @env.delete('DF_TEAMS_CLIENT_ID') + @env.delete('DF_TEAMS_CLIENT_SECRET') + config = UnitHub::Teams::Configuration.new(@env) + assert config.configured_for?(123) + assert_raises(UnitHub::Teams::Configuration::Error) { config.credentials } + end + + def test_missing_student_scope_and_unapproved_publishers_are_rejected + [@mapping_hash.except(:student_visible), @mapping_hash.merge(student_visible: false), + @mapping_hash.merge(publisher_ids: []), @mapping_hash.merge(publisher_ids: ['not-a-guid'])].each do |mapping| + config = UnitHub::Teams::Configuration.new(@env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [mapping].to_json)) + assert_empty config.visible_mapping_keys + assert_raises(UnitHub::Teams::Configuration::Error) { config.mappings } + end + assert_not_requested @token_stub + end + + def test_configuration_rejects_untrusted_paths_tenants_duplicates_and_excess_mappings + [@env.merge('DF_TEAMS_TENANT_ID' => '../common'), + @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash.merge(channel_id: '../../users')].to_json), + @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash, @mapping_hash].to_json), + @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => ([@mapping_hash] * 21).to_json)].each do |values| + assert_empty UnitHub::Teams::Configuration.new(values).visible_mapping_keys + end + end + + def test_tenant_publisher_and_unit_changes_revoke_old_visibility_fingerprint + original = @mapping.key + changed_tenant = @env.merge('DF_TEAMS_TENANT_ID' => CLIENT) + changed_publishers = @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash.merge(publisher_ids: [CLIENT])].to_json) + changed_unit = @env.merge('DF_TEAMS_CHANNEL_MAPPINGS' => [@mapping_hash.merge(unit_id: 124)].to_json) + [changed_tenant, changed_publishers, changed_unit].each do |values| + assert_not_equal original, UnitHub::Teams::Configuration.new(values).mappings.first.key + end + end + + def test_client_credentials_and_two_bounded_pages + next_url = "https://graph.microsoft.com#{PATH}?$skiptoken=next" + first = stub_request(:get, LIST_URL).with(headers: { 'Authorization' => 'Bearer test-access-token' }) + .to_return(body: { value: [{ id: '1' }], '@odata.nextLink' => next_url }.to_json) + second = stub_request(:get, next_url).to_return(body: { value: [{ id: '2' }], '@odata.nextLink' => "https://graph.microsoft.com#{PATH}?$skiptoken=third" }.to_json) + assert_equal %w[1 2], @client.messages(@mapping).pluck('id') + assert_requested @token_stub, times: 1 + assert_requested first, times: 1 + assert_requested second, times: 1 + end + + def test_next_links_cannot_escape_the_configured_channel_or_leak_the_token + ["https://evil.example#{PATH}", 'https://graph.microsoft.com/v1.0/users', + "https://graph.microsoft.com:444#{PATH}", "https://user@graph.microsoft.com#{PATH}", + "https://graph.microsoft.com#{PATH}#fragment", "http://graph.microsoft.com#{PATH}"].each do |url| + stub_request(:get, LIST_URL).to_return(body: { value: [], '@odata.nextLink' => url }.to_json) + assert_raises(UnitHub::Teams::GraphClient::Error) { @client.messages(@mapping) } + end + end + + def test_redirects_and_provider_errors_do_not_expose_body_or_credentials + stub_request(:get, LIST_URL).to_return(status: 302, headers: { 'Location' => 'https://evil.example' }, body: 'private-provider-body') + error = assert_raises(UnitHub::Teams::GraphClient::Error) { @client.messages(@mapping) } + assert_not_includes error.message, 'private-provider-body' + assert_not_includes error.message, 'test-access-token' + assert_requested @token_stub, times: 1 + end + + def test_invalid_json_oversized_body_and_malformed_http_are_sanitized + stub_request(:get, LIST_URL).to_return(body: 'private-provider-body') + assert_raises(UnitHub::Teams::GraphClient::Error) { @client.messages(@mapping) } + stub_request(:get, LIST_URL).to_return(body: 'x' * (2.megabytes + 1)) + assert_raises(UnitHub::Teams::GraphClient::Error) { @client.messages(@mapping) } + stub_request(:get, LIST_URL).to_raise(Net::HTTPBadResponse.new('private-provider-body')) + error = assert_raises(UnitHub::Teams::GraphClient::Error) { @client.messages(@mapping) } + assert_not_includes error.message, 'private-provider-body' + end + + def test_throttling_preserves_bounded_retry_after + stub_request(:get, LIST_URL).to_return(status: 429, headers: { 'Retry-After' => '1800' }) + error = assert_raises(UnitHub::Teams::GraphClient::Throttled) { @client.messages(@mapping) } + assert_equal 1800, error.retry_after + end + + def test_overall_deadline_prevents_network_access + client = UnitHub::Teams::GraphClient.new(@config.credentials, deadline: 0) + assert_raises(UnitHub::Teams::GraphClient::Error) { client.messages(@mapping) } + assert_not_requested @token_stub + end +end