[{"id":"Xck6rcpnzZ4pSTAY2MVa","number":"3255459562132126418","begin":"2025-12-01T08:00:00+00:00","created":"2025-12-02T04:20:49+00:00","end":"2025-12-04T21:23:00+00:00","modified":"2025-12-04T21:28:52+00:00","external_desc":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nGoogle Workspace released a feature to a subset of Workspace customers [Continue your conversations in Google Chat](https://support.google.com/meet/answer/9308979?visit_id=639002449653293387-821172296\u0026p=chat_in_meet\u0026rd=1).\nSome Google Workspace users are experiencing issues with Google Meet host controls for this feature. This may include inability to disable the feature or the ‘disable’ option not being visible under the host controls within Google Calendar meeting invites.\nWe are investigating and will provide an update by Friday, 2025-12-05 10:00 PST with current details.\nWe apologize to all who are affected by this issue.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-12-04T21:28:49+00:00","modified":"2025-12-04T21:28:52+00:00","when":"2025-12-04T21:28:49+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nThe rollback of the \"Continue your conversations in Google Chat\" feature in Google Meet is now complete. This resolves the issues some users experienced, including the inability to disable the feature and the missing toggle in Calendar host controls.\nWe will commence the rollout of this feature again starting Monday, 2025-12-08. No further updates will be provided here. For more detailed information regarding the updated rollout schedule, please refer to the [Google Workspace Updates Blog](https://workspaceupdates.googleblog.com/2025/11/continue-conversations-google-chat-google-meet.html).\nWe apologize for the disruption this may have caused.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-12-02T22:42:20+00:00","modified":"2025-12-04T21:28:49+00:00","when":"2025-12-02T22:42:20+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nGoogle Workspace released a feature to a subset of Workspace customers [Continue your conversations in Google Chat](https://support.google.com/meet/answer/9308979?visit_id=639002449653293387-821172296\u0026p=chat_in_meet\u0026rd=1).\nSome Google Workspace users are experiencing issues with Google Meet host controls for this feature. This may include inability to disable the feature or the ‘disable’ option not being visible under the host controls within Google Calendar meeting invites.\nTo resolve the issues impacting host controls for the \"Continue your conversations in Google Chat\" feature, we are in the process of rolling back this feature. We will provide a further update by Monday, 2025-12-08 10:00 PST.\nWe apologize to all who are affected by this issue.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-12-02T04:20:49+00:00","modified":"2025-12-02T22:42:20+00:00","when":"2025-12-02T04:20:49+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nGoogle Workspace released a feature to a subset of Workspace customers [Continue your conversations in Google Chat](https://support.google.com/meet/answer/9308979?visit_id=639002449653293387-821172296\u0026p=chat_in_meet\u0026rd=1).\nSome Google Workspace users are experiencing issues with Google Meet host controls for this feature. This may include inability to disable the feature or the ‘disable’ option not being visible under the host controls within Google Calendar meeting invites.\nWe are investigating and will provide an update by Friday, 2025-12-05 10:00 PST with current details.\nWe apologize to all who are affected by this issue.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-12-04T21:28:49+00:00","modified":"2025-12-04T21:28:52+00:00","when":"2025-12-04T21:28:49+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nThe rollback of the \"Continue your conversations in Google Chat\" feature in Google Meet is now complete. This resolves the issues some users experienced, including the inability to disable the feature and the missing toggle in Calendar host controls.\nWe will commence the rollout of this feature again starting Monday, 2025-12-08. No further updates will be provided here. For more detailed information regarding the updated rollout schedule, please refer to the [Google Workspace Updates Blog](https://workspaceupdates.googleblog.com/2025/11/continue-conversations-google-chat-google-meet.html).\nWe apologize for the disruption this may have caused.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/Xck6rcpnzZ4pSTAY2MVa"},{"id":"F9Tft7xb3xqenFoKvY4W","number":"3094282244665890171","begin":"2025-11-26T05:34:00+00:00","created":"2025-11-26T07:38:58+00:00","end":"2025-11-26T10:30:00+00:00","modified":"2025-12-03T08:00:57+00:00","external_desc":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-26 02:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","updates":[{"created":"2025-12-02T12:52:38+00:00","modified":"2025-12-03T08:00:57+00:00","when":"2025-12-02T12:52:38+00:00","text":"**# Incident Report**\n**## Summary**\nBetween 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific, Google Meet experienced a service disruption for a total duration of 4 hours and 56 minutes. A subset of Google Meet users in Asia were unable to start or join meetings via the web and on meeting devices.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n**## Root Cause**\nThe incident was caused by an issue within the Meet web frontend server located in the Singapore (SIN) metropolitan area, which is a critical component and responsible for handling Google Meet connections for users in the Asia region.\nThe root cause of the issue was a software update to the Meet web frontend server. The update introduced a blocking call to a dependency in a critical code path, creating a thread deadlock risk.\nApproximately three hours after the update was complete, tasks started deadlocking. The trigger was a combination of regional traffic patterns and latency accessing the dependency involved. This led to stuck requests, Out-Of-Memory (OOM) errors and subsequent task crashes leading to failures when users attempted to load the meeting UI or join meetings.\n**## Remediation and Prevention**\nThe issue started on 25 November 2025 at 21:34 US/Pacific and the problem was first detected by Google's internal alerting systems at 21:41 US/Pacific. Our engineers promptly began working to restore service by taking the following actions in chronological order:\n**25 November, 22:08 US/Pacific** - Shifted traffic away from the most affected Meet web frontend servers.\nEfforts were made to shift traffic away from the specific Meet web frontend servers experiencing the crashes.\n**25 November, 22:20 US/Pacific** - Increase in capacity\nOur engineers increased the resources available to the affected service.\n**25 November, 23:27 US/Pacific** - Rollback of the webfrontend binary\nThe most effective step taken was reverting the software on the Meet web frontend server back to the previous stable version. This action mitigated the issue and restored service availability.\nService was fully restored on **26 November 2025 at 02:30 US/Pacific** after the rollback was complete.\nGoogle is completing the following actions to prevent a repeat of this issue:\n- We are reviewing critical code paths to identify and address potential blocking operations for the dependency that resulted in the deadlocking.\n- We are adding additional checks to prevent adding blocking operations in critical code paths.\n- Increase the load in our canary environments to make it more likely to detect similar issues.\n**## Detailed Description of Impact**\nA subset of Google Meet users in the Asia Pacific region experienced issues loading the meeting user interface (meet.google.com) or were unable to start or join meetings between 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific. The issue primarily affected users accessing Google Meet via Web browsers and Meeting Devices. Mobile users were not affected.","status":"AVAILABLE"},{"created":"2025-11-27T13:49:49+00:00","modified":"2025-12-02T12:53:24+00:00","when":"2025-11-27T13:49:49+00:00","text":"**# Preliminary Incident Report**\nWe apologize for the inconvenience this service disruption may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n**## Summary**\nBetween 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific, Google Meet experienced a service disruption for a total duration of 5 hours and 10 minutes. A subset of Google Meet users in the Asia region were unable to start or join meetings via the web and on meeting devices.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n**## Root Cause**\nThe incident was caused by an issue within the Meet web frontend server located in the Singapore (SIN) metropolitan area, which is a critical component and responsible for handling Google Meet connections for users in the Asia region.\nThe issue began with a software update to the Meet web frontend server. Approximately three hours after this update was complete, a combination of traffic patterns and a code change creating a thread deadlock risk which is currently being investigated further, acted as the trigger for the outage.\nThis trigger led to a sustained period of long-hanging requests in the server's processing queue. This abnormal workload ultimately consumed all available resources, causing the server to experience Out-Of-Memory (OOM) errors and subsequent task crashes. The result (effect) was that the server could not properly process requests, leading to failures when users attempted to load the meeting UI or join meetings.\nFurthermore, the issue detection was slightly delayed because traffic patterns in the immediate interval after the software update did not trigger the issue, delaying our response and mitigation efforts.\n**## Remediation**\nThe issue started on 25 November 2025 at 21:34 US/Pacific and the problem was first detected by Google's internal alerting systems at 21:41 US/Pacific. Our engineers promptly began working to restore service by taking the following actions in chronological order:\n**25 November, 22:08 US/Pacific** - Draining of the most affected cluster\nEfforts were made to shift traffic away from the specific server cluster experiencing the crashes.\n**25 November, 22:20 US/Pacific** - Increase in capacity\nOur engineers increased the resources available to the affected service.\n**25 November, 23:27 US/Pacific** - Rollback of the webfrontend binary\nThe most effective step taken was reverting the software on the Meet web frontend server back to the previous stable version. This action mitigated the issue and restored service availability.\nService was fully restored on **26 November 2025 at 02:30 US/Pacific.**\n**## Detailed Description of Impact**\nA subset of Google Meet users in the Asia Pacific region experienced issues loading the meeting user interface (meet.google.com) or were unable to start or join meetings between 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific. The issue primarily affected users accessing Google Meet via Web browsers and Meeting Devices. Mobile users were not affected.","status":"AVAILABLE"},{"created":"2025-11-26T10:46:48+00:00","modified":"2025-11-27T13:49:49+00:00","when":"2025-11-26T10:46:48+00:00","text":"**Title**\nGoogle Meet customers experienced issues loading meet.google.com in the Asia region.\n**Description**\nThe issue with Google Meet has been resolved for all affected users as of Wednesday, 2025-11-26 02:30 PST.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers experienced issues loading meet.google.com. This issue affected Google Meet traffic in Asia. Other regions were not affected.\n**Workaround**\nNone as the issue has been mitigated.","status":"AVAILABLE"},{"created":"2025-11-26T09:33:54+00:00","modified":"2025-12-02T12:53:39+00:00","when":"2025-11-26T09:33:54+00:00","text":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue and work on a mitigation strategy.\nWe will provide an update by Wednesday, 2025-11-26 03:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-26T08:38:45+00:00","modified":"2025-12-02T12:53:46+00:00","when":"2025-11-26T08:38:45+00:00","text":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue and work on a mitigation strategy.\nWe will provide an update by Wednesday, 2025-11-26 02:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-26T07:38:58+00:00","modified":"2025-12-02T12:53:53+00:00","when":"2025-11-26T07:38:58+00:00","text":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-26 02:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-12-02T12:52:38+00:00","modified":"2025-12-03T08:00:57+00:00","when":"2025-12-02T12:52:38+00:00","text":"**# Incident Report**\n**## Summary**\nBetween 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific, Google Meet experienced a service disruption for a total duration of 4 hours and 56 minutes. A subset of Google Meet users in Asia were unable to start or join meetings via the web and on meeting devices.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n**## Root Cause**\nThe incident was caused by an issue within the Meet web frontend server located in the Singapore (SIN) metropolitan area, which is a critical component and responsible for handling Google Meet connections for users in the Asia region.\nThe root cause of the issue was a software update to the Meet web frontend server. The update introduced a blocking call to a dependency in a critical code path, creating a thread deadlock risk.\nApproximately three hours after the update was complete, tasks started deadlocking. The trigger was a combination of regional traffic patterns and latency accessing the dependency involved. This led to stuck requests, Out-Of-Memory (OOM) errors and subsequent task crashes leading to failures when users attempted to load the meeting UI or join meetings.\n**## Remediation and Prevention**\nThe issue started on 25 November 2025 at 21:34 US/Pacific and the problem was first detected by Google's internal alerting systems at 21:41 US/Pacific. Our engineers promptly began working to restore service by taking the following actions in chronological order:\n**25 November, 22:08 US/Pacific** - Shifted traffic away from the most affected Meet web frontend servers.\nEfforts were made to shift traffic away from the specific Meet web frontend servers experiencing the crashes.\n**25 November, 22:20 US/Pacific** - Increase in capacity\nOur engineers increased the resources available to the affected service.\n**25 November, 23:27 US/Pacific** - Rollback of the webfrontend binary\nThe most effective step taken was reverting the software on the Meet web frontend server back to the previous stable version. This action mitigated the issue and restored service availability.\nService was fully restored on **26 November 2025 at 02:30 US/Pacific** after the rollback was complete.\nGoogle is completing the following actions to prevent a repeat of this issue:\n- We are reviewing critical code paths to identify and address potential blocking operations for the dependency that resulted in the deadlocking.\n- We are adding additional checks to prevent adding blocking operations in critical code paths.\n- Increase the load in our canary environments to make it more likely to detect similar issues.\n**## Detailed Description of Impact**\nA subset of Google Meet users in the Asia Pacific region experienced issues loading the meeting user interface (meet.google.com) or were unable to start or join meetings between 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific. The issue primarily affected users accessing Google Meet via Web browsers and Meeting Devices. Mobile users were not affected.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/F9Tft7xb3xqenFoKvY4W"},{"id":"KZuQxLyBKvPD1zS6sxVA","number":"574347433464346684","begin":"2025-11-24T08:02:00+00:00","created":"2025-11-24T09:47:18+00:00","end":"2025-11-24T09:55:00+00:00","modified":"2025-11-27T10:00:50+00:00","external_desc":"**Summary**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Description**\nWe are experiencing an issue with Google Chat beginning on Monday, 2025-11-24 00:02 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-24 03:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-11-26T23:56:10+00:00","modified":"2025-11-27T10:00:50+00:00","when":"2025-11-26T23:56:10+00:00","text":"# Incident Report\n## Summary\nOn Monday, 24 November 2025, Google Chat experienced an overload which led to service disruptions for users for a duration of 1 hour and 53 minutes. Customers encountered issues including the inability to send messages, load the chat application, and start new conversations. Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.\nWe deeply regret the disruption our Google Workspace customers experienced. We are committed to improving our platform's reliability and performance, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe trigger of the outage was transient errors served from an internal microservice in Europe that caused Chat backend processes to execute a fallback path to retrieve this data. The root cause was that the data retrieved from the fallback path subsequently caused an update to Chat metadata which were sent to user devices, causing those devices to reload data from Chat backend servers, thereby causing more updates to be sent to user devices resulting in a cascading effect that overloaded Chat servers in Europe.\nGoogle engineers have implemented a mitigation to limit high rates of requests to prevent this cascading loop from recurring.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident via alerts and immediately started an investigation.\nTo fully mitigate the impact, our engineering teams took the following steps:\n1. **Increased Server Capacity:** To handle the unexpected surge in traffic, we significantly increased the capacity of our servers in the affected region to ensure the system could process all incoming requests smoothly.\n2. **Limited the Rate of Metadata Updates**: To prevent the cascading loop of requests from overloading Chat servers again, Google engineers have limited the rate of metadata updates so that Chat servers will not be overloaded in this way.\n3. **Paused Non-Essential Background Tasks:** To prioritize the restoration of core chat functions, we temporarily halted several internal background processes. This action helped to reduce the overall load on our systems, freeing up resources to serve user traffic more effectively.\n4. **Gradual Traffic Restoration:** Our automated defenses had initially blocked some traffic to prevent a complete system overload. A crucial part of the recovery was to gradually and carefully allow this traffic back. This controlled approach ensured that as users reconnected, the system wouldn't be overwhelmed again, allowing everyone to successfully re-establish their connection to Google Chat without getting stuck in a retry loop.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* **Fix the immediate cause**: Stop errors in calls to the microservices from triggering metadata updates to devices. The fix prevents a cascading loop of client devices making requests back to Chat servers resulting in overload.\n* **Prevent devices from sending excessive load**: Implement changes in Chat web and mobile apps to prevent them from performing full reload of data from Chat servers when it is not needed. This reloading contributed greatly to the overload of Chat servers.\n* **Improve Monitoring**: Enhance monitoring to identify which specific metadata updates to user devices cause reloads in order to more quickly mitigate similar issues in the future.\n## Detailed Description of Impact\nOn Monday, 24 November 2025 at 00:02 US/Pacific, Google Chat users in Europe experienced a service disruption affecting core features of the service for a duration of 1 hour and 53 minutes.\nAt its peak, the incident affected approximately 20.2% of users in Europe. The issue was resolved by 01:55 US/Pacific.\nUsers encountered problems with the following core functionality:\n* **Initial Application Load**: Many users had trouble loading the Google Chat application.\n* **Opening Chat Rooms**: Users experienced errors when trying to open existing chat rooms.\n* **Sending Messages**: Users faced delays sending messages.\n* I**ncreased Latency**: There was an increase in message delivery latency for affected users.\n* **Notification Rendering**: Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.","status":"AVAILABLE"},{"created":"2025-11-25T18:28:13+00:00","modified":"2025-11-26T23:56:10+00:00","when":"2025-11-25T18:28:13+00:00","text":"# Preliminary Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n## Summary\nOn Monday, 24 November 2025, Google Chat experienced an overload which led to service disruptions for users for a duration of 1 hour and 53 minutes. Customers encountered issues including the inability to send messages, load the chat application, and start new conversations. Some notifications were also incorrectly displayed with \"Anonymous\" as the sender\n## Root Cause\nThe trigger of the outage was transient errors served from an internal microservice in Europe that caused Chat backend processes to execute a fallback path to retrieve this data. The root cause was that the data retrieved from the fallback path subsequently caused an update to Chat metadata which were sent to user devices, causing those devices to reload data from Chat backend servers, thereby causing more updates to be sent to user devices, resulting in a cascading effect that overloaded Chat servers in Europe.\nGoogle engineers have implemented a mitigation to limit high rates of requests to prevent this cascading loop from recurring.\n## Remediation\nGoogle engineers were alerted to the incident via alerts and immediately started an investigation.\nTo fully mitigate the impact, our engineering teams took the following steps:\n1. Increased Server Capacity: To handle the unexpected surge in traffic, we significantly increased the capacity of our servers in the affected region to ensure the system could process all incoming requests smoothly.\n2. Limited the Rate of Metadata Updates: To prevent the cascading loop of requests from overloading Chat servers again, Google engineers have limited the rate of metadata updates so that Chat servers will not be overloaded in this way.\n3. Paused Non-Essential Background Tasks: To prioritize the restoration of core chat functions, we temporarily halted several internal background processes. This action helped to reduce the overall load on our systems, freeing up resources to serve user traffic more effectively.\n4. Gradual Traffic Restoration: Our automated defenses had initially blocked some traffic to prevent a complete system overload. A crucial part of the recovery was to gradually and carefully allow this traffic back. This controlled approach ensured that as users reconnected, the system wouldn't be overwhelmed again, allowing everyone to successfully re-establish their connection to Google Chat without getting stuck in a retry loop.\n## Detailed Description of Impact\nFor a duration of 1 hour and 53 minutes, Google Chat users in Europe experienced a service disruption which manifested as an inability to use core features of the service, such as sending messages and starting conversations.","status":"AVAILABLE"},{"created":"2025-11-24T10:06:57+00:00","modified":"2025-11-25T18:28:13+00:00","when":"2025-11-24T10:06:57+00:00","text":"**Summary**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Description**\nThe issue with Google Chat has been resolved for all affected users as of Monday, 2025-11-24 01:55 PST.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-24T09:47:18+00:00","modified":"2025-11-24T10:06:57+00:00","when":"2025-11-24T09:47:18+00:00","text":"**Summary**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Description**\nWe are experiencing an issue with Google Chat beginning on Monday, 2025-11-24 00:02 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-24 03:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-11-26T23:56:10+00:00","modified":"2025-11-27T10:00:50+00:00","when":"2025-11-26T23:56:10+00:00","text":"# Incident Report\n## Summary\nOn Monday, 24 November 2025, Google Chat experienced an overload which led to service disruptions for users for a duration of 1 hour and 53 minutes. Customers encountered issues including the inability to send messages, load the chat application, and start new conversations. Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.\nWe deeply regret the disruption our Google Workspace customers experienced. We are committed to improving our platform's reliability and performance, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe trigger of the outage was transient errors served from an internal microservice in Europe that caused Chat backend processes to execute a fallback path to retrieve this data. The root cause was that the data retrieved from the fallback path subsequently caused an update to Chat metadata which were sent to user devices, causing those devices to reload data from Chat backend servers, thereby causing more updates to be sent to user devices resulting in a cascading effect that overloaded Chat servers in Europe.\nGoogle engineers have implemented a mitigation to limit high rates of requests to prevent this cascading loop from recurring.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident via alerts and immediately started an investigation.\nTo fully mitigate the impact, our engineering teams took the following steps:\n1. **Increased Server Capacity:** To handle the unexpected surge in traffic, we significantly increased the capacity of our servers in the affected region to ensure the system could process all incoming requests smoothly.\n2. **Limited the Rate of Metadata Updates**: To prevent the cascading loop of requests from overloading Chat servers again, Google engineers have limited the rate of metadata updates so that Chat servers will not be overloaded in this way.\n3. **Paused Non-Essential Background Tasks:** To prioritize the restoration of core chat functions, we temporarily halted several internal background processes. This action helped to reduce the overall load on our systems, freeing up resources to serve user traffic more effectively.\n4. **Gradual Traffic Restoration:** Our automated defenses had initially blocked some traffic to prevent a complete system overload. A crucial part of the recovery was to gradually and carefully allow this traffic back. This controlled approach ensured that as users reconnected, the system wouldn't be overwhelmed again, allowing everyone to successfully re-establish their connection to Google Chat without getting stuck in a retry loop.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* **Fix the immediate cause**: Stop errors in calls to the microservices from triggering metadata updates to devices. The fix prevents a cascading loop of client devices making requests back to Chat servers resulting in overload.\n* **Prevent devices from sending excessive load**: Implement changes in Chat web and mobile apps to prevent them from performing full reload of data from Chat servers when it is not needed. This reloading contributed greatly to the overload of Chat servers.\n* **Improve Monitoring**: Enhance monitoring to identify which specific metadata updates to user devices cause reloads in order to more quickly mitigate similar issues in the future.\n## Detailed Description of Impact\nOn Monday, 24 November 2025 at 00:02 US/Pacific, Google Chat users in Europe experienced a service disruption affecting core features of the service for a duration of 1 hour and 53 minutes.\nAt its peak, the incident affected approximately 20.2% of users in Europe. The issue was resolved by 01:55 US/Pacific.\nUsers encountered problems with the following core functionality:\n* **Initial Application Load**: Many users had trouble loading the Google Chat application.\n* **Opening Chat Rooms**: Users experienced errors when trying to open existing chat rooms.\n* **Sending Messages**: Users faced delays sending messages.\n* I**ncreased Latency**: There was an increase in message delivery latency for affected users.\n* **Notification Rendering**: Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"Ht9Vx5PFzDPHYvrnrvSk","service_name":"Google Chat","affected_products":[{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/KZuQxLyBKvPD1zS6sxVA"},{"id":"EzZso1djQQfwqEmiJuDi","number":"5127096292705077","begin":"2025-11-14T18:05:00+00:00","created":"2025-11-17T02:29:25+00:00","end":"2025-11-17T03:10:00+00:00","modified":"2025-11-17T04:12:20+00:00","external_desc":"**Summary:**\nA subset of customers may be experiencing UI issue when files are uploaded via Chat to Google Workspace Support.\n**Description:**\nWe are experiencing an intermittent issue with Google Workspace Chat Support beginning at Friday, 2025-11-14 10:05 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Sunday, 2025-11-16 20:30 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nA subset Customers using Chat to Google Workspace Support might encounter a UI issue where uploaded files appear as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files are successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-11-17T04:11:45+00:00","modified":"2025-11-17T04:12:20+00:00","when":"2025-11-17T04:11:45+00:00","text":"**Summary:**\nA subset of customers might have experienced UI issue while uploading the files to Google Support via Chat.\n**Description:**\nThe issue with Google Cloud Support has been resolved for all affected users as of Sunday, 2025-11-16 19:10 PST.\nThe issue was due to a recent code change. Our engineers have mitigated the issue by rolling back the system to a previous version.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nA subset of customers using Chat to Google Support might have encountered a UI issue where uploaded files appeared as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files were successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNot applicable as the issue has been resolved.\nNone at this time.","status":"AVAILABLE"},{"created":"2025-11-17T02:29:25+00:00","modified":"2025-11-17T04:11:45+00:00","when":"2025-11-17T02:29:25+00:00","text":"**Summary:**\nA subset of customers may be experiencing UI issue when files are uploaded via Chat to Google Workspace Support.\n**Description:**\nWe are experiencing an intermittent issue with Google Workspace Chat Support beginning at Friday, 2025-11-14 10:05 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Sunday, 2025-11-16 20:30 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nA subset Customers using Chat to Google Workspace Support might encounter a UI issue where uploaded files appear as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files are successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-11-17T04:11:45+00:00","modified":"2025-11-17T04:12:20+00:00","when":"2025-11-17T04:11:45+00:00","text":"**Summary:**\nA subset of customers might have experienced UI issue while uploading the files to Google Support via Chat.\n**Description:**\nThe issue with Google Cloud Support has been resolved for all affected users as of Sunday, 2025-11-16 19:10 PST.\nThe issue was due to a recent code change. Our engineers have mitigated the issue by rolling back the system to a previous version.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nA subset of customers using Chat to Google Support might have encountered a UI issue where uploaded files appeared as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files were successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNot applicable as the issue has been resolved.\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/EzZso1djQQfwqEmiJuDi"},{"id":"viWmkGEagnWrqYfb7VpS","number":"2706164284614039912","begin":"2025-11-12T17:00:00+00:00","created":"2025-11-12T18:44:45+00:00","end":"2025-11-12T21:45:00+00:00","modified":"2025-11-13T01:23:33+00:00","external_desc":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 11:10 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-11-13T01:23:33+00:00","modified":"2025-11-13T01:23:33+00:00","when":"2025-11-13T01:23:33+00:00","text":"**Title**\nAccess issues with Workspace products are now mitigated.\n**Description**\nThe issue with Google Sheets, Google Docs, Google Slides, and Google Forms has been resolved for all affected users as of Wednesday, 2025-11-12 13:45 PST.\nThe root cause was associated with a third party and external to Google infrastructure.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms**\nA small number of customers may have encountered SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nNot applicable.","status":"AVAILABLE"},{"created":"2025-11-12T23:32:28+00:00","modified":"2025-11-13T01:23:33+00:00","when":"2025-11-12T23:32:28+00:00","text":"**Title**\nWe are mitigating the access issues with multiple workspace products.\n**Description**\nOur engineering team has identified the root cause. We are continuing to see a decrease in user impact and are currently working to resolve the issue.\nWe will provide an update by Wednesday, 2025-11-12 16:30 PST with current details.\n**Customer Symptoms**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nAffected customers may try accessing via a mobile device.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T22:24:14+00:00","modified":"2025-11-12T23:33:36+00:00","when":"2025-11-12T22:24:14+00:00","text":"**Title**\nWe are mitigating the access issues with multiple workspace products.\n**Description**\nOur engineering team has identified the root cause. We are seeing a decrease in user impact and are currently working to resolve the issue.\nWe will provide an update by Wednesday, 2025-11-12 15:30 PST with current details.\n**Customer Symptoms**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nAffected customers may try accessing via a mobile device.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T21:18:06+00:00","modified":"2025-11-12T23:33:51+00:00","when":"2025-11-12T21:18:06+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs, Google Forms, Google Slides beginning at Wednesday, 2025-11-12 09:00 PST.\nOur engineering team is actively investigating the issue.\nWe will provide an update by Wednesday, 2025-11-12 14:15 PST with current details.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround:**\nImpacted customers may try and reboot their routers as a potential workaround.\nCustomers may also try to access via a mobile device..","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T20:57:01+00:00","modified":"2025-11-12T21:18:06+00:00","when":"2025-11-12T20:57:01+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs, Google Slides, Google Forms beginning at Wednesday, 2025-11-12 09:00 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 13:15 PST with current details.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T20:08:52+00:00","modified":"2025-11-12T20:57:01+00:00","when":"2025-11-12T20:08:52+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 12:40 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T19:46:18+00:00","modified":"2025-11-12T20:08:52+00:00","when":"2025-11-12T19:46:18+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 12:10 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-11-12T19:05:14+00:00","modified":"2025-11-12T19:46:18+00:00","when":"2025-11-12T19:05:14+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 11:45 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-11-12T18:44:45+00:00","modified":"2025-11-12T19:05:14+00:00","when":"2025-11-12T18:44:45+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 11:10 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-11-13T01:23:33+00:00","modified":"2025-11-13T01:23:33+00:00","when":"2025-11-13T01:23:33+00:00","text":"**Title**\nAccess issues with Workspace products are now mitigated.\n**Description**\nThe issue with Google Sheets, Google Docs, Google Slides, and Google Forms has been resolved for all affected users as of Wednesday, 2025-11-12 13:45 PST.\nThe root cause was associated with a third party and external to Google infrastructure.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms**\nA small number of customers may have encountered SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nNot applicable.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Forms","id":"3mtqLXm6QxeM6bAVkVez"},{"title":"Google Sheets","id":"GsNjeYHsogeRgTiC5coU"},{"title":"Google Slides","id":"DvWBgkXVhodA3WYwgZoB"}],"uri":"incidents/viWmkGEagnWrqYfb7VpS"},{"id":"tR2hwYEXUcAjWVfy1kp8","number":"16997046130160050708","begin":"2025-11-10T14:00:00+00:00","created":"2025-11-10T15:50:36+00:00","end":"2025-11-10T16:58:00+00:00","modified":"2025-11-10T17:26:28+00:00","external_desc":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nWe are experiencing an issue with Gmail beginning at Monday, 2025-11-10 06:19 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-10 09:00 PST with current details.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-11-10T17:26:12+00:00","modified":"2025-11-10T17:26:28+00:00","when":"2025-11-10T17:26:12+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nThe issue with Gmail has been resolved for all affected users as of Monday, 2025-11-10 08:58 US/Pacific.\nThe issue was due to network latency which eventually translated into errors. Our engineers have mitigated the issue by diverting the traffic to other regions.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-11-10T16:43:08+00:00","modified":"2025-11-10T17:26:12+00:00","when":"2025-11-10T16:43:08+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe are seeing significant improvement and the error rates have subsided.\nWe will provide an update by Monday, 2025-11-10 10:00 US/Pacific with current details.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-11-10T15:50:36+00:00","modified":"2025-11-10T16:43:08+00:00","when":"2025-11-10T15:50:36+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nWe are experiencing an issue with Gmail beginning at Monday, 2025-11-10 06:19 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-10 09:00 PST with current details.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-11-10T17:26:12+00:00","modified":"2025-11-10T17:26:28+00:00","when":"2025-11-10T17:26:12+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nThe issue with Gmail has been resolved for all affected users as of Monday, 2025-11-10 08:58 US/Pacific.\nThe issue was due to network latency which eventually translated into errors. Our engineers have mitigated the issue by diverting the traffic to other regions.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/tR2hwYEXUcAjWVfy1kp8"},{"id":"kAS3CYqGhecvgBGnzecT","number":"18333820535903255874","begin":"2025-11-05T05:05:00+00:00","created":"2025-11-05T06:07:01+00:00","end":"2025-11-05T05:55:00+00:00","modified":"2025-11-10T21:31:15+00:00","external_desc":"**Title**\nGoogle Chat customers are experiencing issues loading Chat, creating DMs (Direct Messages) and Chat Spaces.\n**Description**\nWe are experiencing an issue with Google Chat , beginning on Tuesday, 2025-11-04 21:05 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Tuesday, 2025-11-04 22:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may experience issues while loading Chat, creating DMs (Direct Messages) and Chat Spaces, and may experience high latency while sending and receiving messages.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-11-10T21:31:15+00:00","modified":"2025-11-10T21:31:15+00:00","when":"2025-11-10T21:31:15+00:00","text":"# Incident Report\n## Summary\nGoogle Chat users experienced errors when creating new direct messages (DMs) and chat spaces for a total duration of 50 minutes between 21:05 US/Pacific to 21:55 US/Pacific on Tuesday November 4, 2025. At its peak, the issue impacted approximately 0.3% of users. Affected users also observed increased latency in message delivery and issues with initial application loading and opening chat rooms.\nTo our Google Chat users who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident was triggered by database lock contention and hotspotting caused by a large volume of automated background jobs for database maintenance. These resource-intensive tasks created a bottleneck. This resulted in a cascading failure where other unrelated operations that relied on database resources were delayed, causing yet more operations to be delayed in a similar way. This resulted in the errors that users experienced when trying to perform actions like creating new conversations.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident through a series of monitoring alerts at 21:06 US/Pacific on Tuesday November 4, 2025 for high error rates on our internal chat services, and immediately started to investigate.\nThe primary mitigation action was to temporarily halt the queue that processes these background cleanup jobs at 21:45 US/Pacific. This action immediately reduced the load and relieved hotspotting on the database, allowing normal operations to resume. A more specific control was later introduced to disable only the problematic background task on November 5, 2025 at 17:34 US/Pacific.\nWe apologize for the disruption this incident caused. We are committed to learning from this event and implementing changes to prevent similar issues in the future.\nThe following preventative measures are planned and will be implemented:\n- **Improve Monitoring:** Enhance monitoring to quickly identify the specific background tasks causing database contention.\n- **Improve Resilience:** Improve and document procedures for our engineering teams to mitigate similar incidents more quickly, such as by temporarily pausing or slowing down non-essential background tasks.\n- **Fix Underlying Cause:** Implement a fix to prevent root cause from occurring again by reducing the excessive resource usage during background cleanup jobs and to prevent them from overloading the system and cascading to other operations in the future.\n## Detailed Description of Impact\nThe incident had a wide-ranging impact on Google Chat users and connected services, with the most significant effects detailed below:\nAt its peak, the incident affected approximately 0.3% of users. The primary and most direct impact was on core chat functionalities, where users experienced errors when creating new direct messages (DMs) and chat spaces.\n- In addition to the primary issue, users also encountered problems with:\n- Initial Application Load: Many users had trouble loading the Google Chat application.\n- Opening Chat Rooms: Users experienced errors when trying to open existing chat rooms.\n- Sending Messages: Users faced delays sending messages.\n- Increased Latency: There was an increase in message delivery latency for affected users.","status":"AVAILABLE"},{"created":"2025-11-05T10:14:05+00:00","modified":"2025-11-10T21:31:15+00:00","when":"2025-11-05T10:14:05+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 4 November, 2025 21:05\n**Incident End:** 4 November, 2025 21:55\n**Duration:** 50 minutes\n**Affected Services and Features:**\nGoogle Chat\n**Description:**\nGoogle Chat users experienced issues when loading Chat, creating and opening DMs (Direct Messages) and Chat Spaces for a duration of 50 minutes. From preliminary analysis the root cause was a temporary contention in the underlying storage system.\n**Customer Impact:**\nAffected users faced error when trying to load Google Chat, start new direct messages, or create/open chat spaces.","status":"AVAILABLE"},{"created":"2025-11-05T06:19:40+00:00","modified":"2025-11-05T10:14:05+00:00","when":"2025-11-05T06:19:40+00:00","text":"The issue with Google Chat has been resolved for all affected users as of Tuesday, 2025-11-04 21:45 PST.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-05T06:07:01+00:00","modified":"2025-11-05T06:19:40+00:00","when":"2025-11-05T06:07:01+00:00","text":"**Title**\nGoogle Chat customers are experiencing issues loading Chat, creating DMs (Direct Messages) and Chat Spaces.\n**Description**\nWe are experiencing an issue with Google Chat , beginning on Tuesday, 2025-11-04 21:05 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Tuesday, 2025-11-04 22:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may experience issues while loading Chat, creating DMs (Direct Messages) and Chat Spaces, and may experience high latency while sending and receiving messages.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-11-10T21:31:15+00:00","modified":"2025-11-10T21:31:15+00:00","when":"2025-11-10T21:31:15+00:00","text":"# Incident Report\n## Summary\nGoogle Chat users experienced errors when creating new direct messages (DMs) and chat spaces for a total duration of 50 minutes between 21:05 US/Pacific to 21:55 US/Pacific on Tuesday November 4, 2025. At its peak, the issue impacted approximately 0.3% of users. Affected users also observed increased latency in message delivery and issues with initial application loading and opening chat rooms.\nTo our Google Chat users who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident was triggered by database lock contention and hotspotting caused by a large volume of automated background jobs for database maintenance. These resource-intensive tasks created a bottleneck. This resulted in a cascading failure where other unrelated operations that relied on database resources were delayed, causing yet more operations to be delayed in a similar way. This resulted in the errors that users experienced when trying to perform actions like creating new conversations.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident through a series of monitoring alerts at 21:06 US/Pacific on Tuesday November 4, 2025 for high error rates on our internal chat services, and immediately started to investigate.\nThe primary mitigation action was to temporarily halt the queue that processes these background cleanup jobs at 21:45 US/Pacific. This action immediately reduced the load and relieved hotspotting on the database, allowing normal operations to resume. A more specific control was later introduced to disable only the problematic background task on November 5, 2025 at 17:34 US/Pacific.\nWe apologize for the disruption this incident caused. We are committed to learning from this event and implementing changes to prevent similar issues in the future.\nThe following preventative measures are planned and will be implemented:\n- **Improve Monitoring:** Enhance monitoring to quickly identify the specific background tasks causing database contention.\n- **Improve Resilience:** Improve and document procedures for our engineering teams to mitigate similar incidents more quickly, such as by temporarily pausing or slowing down non-essential background tasks.\n- **Fix Underlying Cause:** Implement a fix to prevent root cause from occurring again by reducing the excessive resource usage during background cleanup jobs and to prevent them from overloading the system and cascading to other operations in the future.\n## Detailed Description of Impact\nThe incident had a wide-ranging impact on Google Chat users and connected services, with the most significant effects detailed below:\nAt its peak, the incident affected approximately 0.3% of users. The primary and most direct impact was on core chat functionalities, where users experienced errors when creating new direct messages (DMs) and chat spaces.\n- In addition to the primary issue, users also encountered problems with:\n- Initial Application Load: Many users had trouble loading the Google Chat application.\n- Opening Chat Rooms: Users experienced errors when trying to open existing chat rooms.\n- Sending Messages: Users faced delays sending messages.\n- Increased Latency: There was an increase in message delivery latency for affected users.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"Ht9Vx5PFzDPHYvrnrvSk","service_name":"Google Chat","affected_products":[{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/kAS3CYqGhecvgBGnzecT"},{"id":"AqnPb3cc3JPV96CpxGgL","number":"12714380912420475217","begin":"2025-10-30T15:05:00+00:00","created":"2025-10-30T16:19:13+00:00","end":"2025-10-31T11:56:00+00:00","modified":"2025-10-31T12:42:24+00:00","external_desc":"**Summary**\nGoogle Meet meetings are seeing elevated error rates in upgrading the call on Android.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures to upgrade the call.\nOur engineering team has identified the root cause and is working on mitigation.\nWe will provide more information by Thursday, 2025-10-30 15:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are seeing high error rates in upgrading the call\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","updates":[{"created":"2025-10-31T12:41:33+00:00","modified":"2025-10-31T12:42:24+00:00","when":"2025-10-31T12:41:33+00:00","text":"**Summary**\nElevated error rates observed for Android users when upgrading calls to Google Meet meetings.\n**Description**\nThe issue with Google Meet has been resolved for all affected customers as of Friday, 2025-10-31 04:56 US/Pacific.\nFrom preliminary analysis, the issue was triggered by a recent UI change that made a feature for upgrading calls to Google Meet more prominent for Android users. This led to a surge in usage and revealed an underlying issue with the feature, causing a high failure rate when upgrading calls.\nOur engineers mitigated the issue by rolling back the UI change. The team will continue to investigate and address the underlying deficiencies in the feature.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings were observing high error rates when upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"AVAILABLE"},{"created":"2025-10-30T22:50:52+00:00","modified":"2025-10-31T12:41:33+00:00","when":"2025-10-30T22:50:52+00:00","text":"**Summary**\nElevated error rates observed for Android users when upgrading calls to Google Meet meetings.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures when upgrading the call.\nOur engineering team has identified the underlying cause of the issue and a rollback was initiated to mitigate impact. Based on the progress of the rollback, we are observing positive results pertaining to recovery.\nWe expect full mitigation to be completed by Friday, 2025-10-31 12:00 US/Pacific.\nWe will provide more information by Friday, 2025-10-31 12:30 US/Pacific with current details.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are observing high error rates when upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"SERVICE_INFORMATION"},{"created":"2025-10-30T22:00:54+00:00","modified":"2025-10-30T22:50:52+00:00","when":"2025-10-30T22:00:54+00:00","text":"**Summary**\nGoogle Meet meetings are seeing elevated error rates in upgrading the call on Android.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures to upgrade the call.\nOur engineering team is actively working on the mitigation efforts.\nWe will provide more information by Thursday, 2025-10-30 16:00 US/Pacific with current details.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are seeing high error rates in upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"SERVICE_INFORMATION"},{"created":"2025-10-30T16:19:13+00:00","modified":"2025-10-30T22:00:54+00:00","when":"2025-10-30T16:19:13+00:00","text":"**Summary**\nGoogle Meet meetings are seeing elevated error rates in upgrading the call on Android.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures to upgrade the call.\nOur engineering team has identified the root cause and is working on mitigation.\nWe will provide more information by Thursday, 2025-10-30 15:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are seeing high error rates in upgrading the call\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-31T12:41:33+00:00","modified":"2025-10-31T12:42:24+00:00","when":"2025-10-31T12:41:33+00:00","text":"**Summary**\nElevated error rates observed for Android users when upgrading calls to Google Meet meetings.\n**Description**\nThe issue with Google Meet has been resolved for all affected customers as of Friday, 2025-10-31 04:56 US/Pacific.\nFrom preliminary analysis, the issue was triggered by a recent UI change that made a feature for upgrading calls to Google Meet more prominent for Android users. This led to a surge in usage and revealed an underlying issue with the feature, causing a high failure rate when upgrading calls.\nOur engineers mitigated the issue by rolling back the UI change. The team will continue to investigate and address the underlying deficiencies in the feature.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings were observing high error rates when upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/AqnPb3cc3JPV96CpxGgL"},{"id":"qgaHmpt74UC6CrdZoGmc","number":"10845080150737297681","begin":"2025-10-25T06:05:00+00:00","created":"2025-10-25T07:19:50+00:00","end":"2025-10-25T08:00:00+00:00","modified":"2025-10-27T04:15:53+00:00","external_desc":"**Summary**\nAppSheet customers may experience latency issues\n**Description**\nWe are experiencing an issue with AppSheet beginning at Friday, 2025-10-24 23:05 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Saturday, 2025-10-25 01:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nThe impacted customers may experience failures while loading apps or writing data using AppSheet.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-10-27T04:15:53+00:00","modified":"2025-10-27T04:15:54+00:00","when":"2025-10-27T04:15:53+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 24 October, 2025 23:05\n**Incident End:** 25 October, 2025 01:00\n**Duration:** 1 hour 55 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:**\nGlobal\n**Description:**\nStarting on 24 October 23:05 US/Pacific, AppSheet experienced failures while loading apps or writing data for a duration of 1 hour and 55 minutes.\nFrom preliminary analysis, the issue was caused due to a maintenance event on a database resulting in lock contention. Our engineering mitigated the impact by manually resolving the contention issue.\n**Customer Impact:**\nThe impacted customers might have experienced failures while loading apps or writing data using AppSheet.","status":"AVAILABLE"},{"created":"2025-10-25T08:24:53+00:00","modified":"2025-10-27T04:15:53+00:00","when":"2025-10-25T08:24:53+00:00","text":"**Summary**\nAppSheet customers might have experienced latency issues\n**Description**\nThe issue with AppSheet has been resolved for all affected users as of Saturday, 2025-10-25 01:00 PDT.\nThe issue was triggered due to a maintenance event on a database resulting in lock contention and our engineering mitigated the impact by manually resolving the contention issue.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nThe impacted customers might have experienced failures while loading apps or writing data using AppSheet.\n**Workaround**\nNone","status":"AVAILABLE"},{"created":"2025-10-25T07:19:50+00:00","modified":"2025-10-25T08:24:53+00:00","when":"2025-10-25T07:19:50+00:00","text":"**Summary**\nAppSheet customers may experience latency issues\n**Description**\nWe are experiencing an issue with AppSheet beginning at Friday, 2025-10-24 23:05 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Saturday, 2025-10-25 01:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nThe impacted customers may experience failures while loading apps or writing data using AppSheet.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-27T04:15:53+00:00","modified":"2025-10-27T04:15:54+00:00","when":"2025-10-27T04:15:53+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 24 October, 2025 23:05\n**Incident End:** 25 October, 2025 01:00\n**Duration:** 1 hour 55 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:**\nGlobal\n**Description:**\nStarting on 24 October 23:05 US/Pacific, AppSheet experienced failures while loading apps or writing data for a duration of 1 hour and 55 minutes.\nFrom preliminary analysis, the issue was caused due to a maintenance event on a database resulting in lock contention. Our engineering mitigated the impact by manually resolving the contention issue.\n**Customer Impact:**\nThe impacted customers might have experienced failures while loading apps or writing data using AppSheet.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/qgaHmpt74UC6CrdZoGmc"},{"id":"Jxvg9KeHUPnwfQ4nPgiT","number":"15026016134995879214","begin":"2025-10-24T17:45:00+00:00","created":"2025-10-25T00:15:53+00:00","end":"2025-10-25T19:05:00+00:00","modified":"2025-10-27T23:44:42+00:00","external_desc":"**Summary**\nA subset of customers in EDU domains are experiencing issues accessing Gmail mobile applications.\n**Description**\nWe are experiencing an issue with Gmail beginning on Friday, 2025-10-24 10:45 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-10-24 20:30 PDT with current details.\n**Symptoms**\nAffected customers are experiencing issues accessing Gmail mobile applications. Gmail web access is unaffected.\n**Workaround**\nCustomers can access Gmail from the web browser.","updates":[{"created":"2025-10-27T23:44:42+00:00","modified":"2025-10-27T23:44:43+00:00","when":"2025-10-27T23:44:42+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service issue may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 24 October 2025, 10:45\n**Incident End:** 25 October 2025, 12:05\n**Duration:** 25 hours, 20 minutes\n**Affected Services and Features:**\nGmail - Mobile application users on both ios and android\n**Description:**\nSome Workspace for Education users reported experiencing issues accessing Gmail on their mobile devices.\nPreliminary analysis indicates that the issue was caused by a recent code change in the UX stack of our first-party mobile applications. Our engineering team mitigated the impact by rolling back the problematic changes.\n**Customer Impact:**\nAffected users were unable to access Gmail through mobile applications on Android and iOS devices. Gmail web access was unaffected during the issue.","status":"AVAILABLE"},{"created":"2025-10-26T02:18:18+00:00","modified":"2025-10-27T23:44:42+00:00","when":"2025-10-26T02:18:18+00:00","text":"**Summary**\nA subset of customers in EDU domains experienced issues accessing Gmail mobile applications.\n**Description**\nThe issue with the GMail is resolved for all the users on Saturday, 2025-10-25 12:05 PDT\nAs per our preliminary analysis, the issue was caused due to a recent code change introduced to OneGoogle for Android and iOS apps. Our engineering team mitigated the issue by rolling back these problematic changes.\nWe apologize to all who are affected by the disruption. We thank you for your patience while we worked on resolving the issue.","status":"SERVICE_INFORMATION"},{"created":"2025-10-25T02:55:32+00:00","modified":"2025-10-26T02:18:18+00:00","when":"2025-10-25T02:55:32+00:00","text":"**Summary**\nA subset of customers in EDU domains are experiencing issues accessing Gmail mobile applications.\n**Description**\nWe are experiencing an issue with Gmail beginning on Friday, 2025-10-24 10:45 PDT.\nOur engineering team has identified the root cause and implemented mitigation measures. And, currently we are monitoring the status of mitigation.\nWe will provide an update by Monday, 2025-10-27 11:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nAffected customers may have experienced issues accessing Gmail mobile applications. Gmail web access is unaffected.\n**Workaround**\nCustomers can access Gmail from the web browser.","status":"SERVICE_INFORMATION"},{"created":"2025-10-25T00:15:53+00:00","modified":"2025-10-25T02:55:32+00:00","when":"2025-10-25T00:15:53+00:00","text":"**Summary**\nA subset of customers in EDU domains are experiencing issues accessing Gmail mobile applications.\n**Description**\nWe are experiencing an issue with Gmail beginning on Friday, 2025-10-24 10:45 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-10-24 20:30 PDT with current details.\n**Symptoms**\nAffected customers are experiencing issues accessing Gmail mobile applications. Gmail web access is unaffected.\n**Workaround**\nCustomers can access Gmail from the web browser.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-27T23:44:42+00:00","modified":"2025-10-27T23:44:43+00:00","when":"2025-10-27T23:44:42+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service issue may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 24 October 2025, 10:45\n**Incident End:** 25 October 2025, 12:05\n**Duration:** 25 hours, 20 minutes\n**Affected Services and Features:**\nGmail - Mobile application users on both ios and android\n**Description:**\nSome Workspace for Education users reported experiencing issues accessing Gmail on their mobile devices.\nPreliminary analysis indicates that the issue was caused by a recent code change in the UX stack of our first-party mobile applications. Our engineering team mitigated the impact by rolling back the problematic changes.\n**Customer Impact:**\nAffected users were unable to access Gmail through mobile applications on Android and iOS devices. Gmail web access was unaffected during the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/Jxvg9KeHUPnwfQ4nPgiT"},{"id":"UYc9QNUSG1gniUUcDkDy","number":"18211409613549623115","begin":"2025-10-20T19:41:00+00:00","created":"2025-10-20T19:50:06+00:00","end":"2025-10-24T00:19:00+00:00","modified":"2025-10-27T11:10:45+00:00","external_desc":"**Summary**\nWe are investigating an issue impacting Endpoint management for Windows Devices\n**Description**\nWe are experiencing an issue with Endpoint management beginning on Wednesday, 2025-10-01 00:00 PDT. We detected the impact progressively increasing on Tuesday, 2025-10-14 10:23 PDT.\nThe Windows native settings app responsible for communicating with Google Identity and Endpoint management is using Internet Explorer 11 which has been deprecated by Microsoft. Google Identity servers are failing to serve requests for IE11 which is impacting device enrollment.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-10-20 15:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time","updates":[{"created":"2025-10-27T11:10:45+00:00","modified":"2025-10-27T11:10:46+00:00","when":"2025-10-27T11:10:45+00:00","text":"# Incident Report\n## Summary\nBeginning on 1 October 2025, Google Endpoint Management customers experienced issues enrolling new Windows devices using the native Windows Settings application or direct enrollment deeplinks. This incident started on 1 October 2025 at 00:00 US/Pacific and lasted for 22 days, 17 hours and 19 minutes, with resolution on 23 October 2025 at 17:19 US/Pacific. We sincerely apologize for any impact this disruption had on your device management workflows. We are taking measures to prevent a recurrence.\n## Root Cause\nThe root cause of this incident was a deprecated[1] browser compatibility issue. The device enrollment process initiated from the Windows Settings application relies on an embedded browser component. This component's behavior is based on the now deprecated Internet Explorer 11. Recent updates to Google's Identity servers, intended to enhance security and modernize the platform, inadvertently removed compatibility with the outdated protocols and behaviors exhibited by the Internet Explorer 11 engine. As a result, authentication requests from the Windows Settings enrollment flow could no longer be processed successfully by the updated Identity servers, causing the enrollment to fail.\n[1] Deprecated browser compatibility issue: https://learn.microsoft.com/en-us/lifecycle/products/internet-explorer-11\n## Remediation and Prevention\nThe issue was detected on Tuesday, 14 October 2025, at 10:23 US/Pacific, as the impact progressively increased, prompting a detailed investigation. Google engineers traced the failures back to the incompatibility between the Google Identity servers and the enrollment process embedded within the Windows Settings application, which uses a deprecated Internet Explorer 11 component.\nTo remediate the issue, Google engineers developed and deployed a targeted fix to the Google Identity servers. This update restored the servers' ability to handle authentication requests originating from the enrollment flow dependent on the Internet Explorer 11 technology. The rollout of this fix was completed on 23 October 2025 at 17:19 US/Pacific, fully restoring the ability to enroll new Windows devices via the Windows Settings application and deeplinks.\nGoogle is committed to preventing similar incidents in the future and is completing the following actions:\nWe are enhancing our testing procedures for Windows device enrollment flows.\nWe are exploring improvements in monitoring to detect issues on lower-traffic enrollment pathways more rapidly.\n## Detailed Description of Impact\nGoogle Endpoint Management - From 1 October 2025, 00:00 US/Pacific to 23 October 2025, 17:19 US/Pacific, customers were unable to enroll new Windows devices into Google Endpoint Management when using the built-in Windows Settings application (as described in “Enroll a device” in Windows device management) or an enrollment deeplink. The enrollment process would fail during the authentication phase due to the incompatibility.\nCustomers using the Google Credential Provider for Windows (GCPW) for device setup and login (as described in “Install Google Credential Provider for Windows”) were not affected by this incident and could continue to enroll and manage devices without issue. The impact was limited to the specific enrollment method relying on the Windows Settings interface.","status":"AVAILABLE"},{"created":"2025-10-24T06:50:16+00:00","modified":"2025-10-27T11:10:45+00:00","when":"2025-10-24T06:50:16+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 01 October, 2025 00:00\n**Incident End:** 23 October, 2025 17:19\n**Duration:** 22 days, 17 hours, 19 minutes\n**Affected Services and Features:**\nEndpoint Management\n**Description:**\nStarting on 01 October 2025, Endpoint Management customers experienced the inability to enroll new Windows devices using Windows Settings app or deeplink. We detected the impact when it progressively increased on Tuesday, 2025-10-14 10:23 PDT.\nFrom our analysis, the issue originated from a browser compatibility concern. Specifically, the Microsoft-provided Windows Settings application utilized the deprecated Internet Explorer 11, rendering it incompatible with recent modifications to the identity servers. Our engineers resolved this by deploying a fix to the Google Identity servers to restore support for these requests.\n**Customer Impact:**\nThe Windows native settings application, which facilitates communication with Google Identity and Endpoint management, relies on Internet Explorer 11, a browser no longer supported by Microsoft. Device enrollment was affected due to Google Identity servers unable to fulfill requests from Internet Explorer 11.\nCustomers encountered issues enrolling new Windows devices in Google Endpoint management through the Windows Settings app or deeplink1. However, users leveraging Google Credential Provider for Windows (GCPW) were unaffected by this problem2.\n**Reference(s):**\n1 [Enroll a device in Windows device management](https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0%29*%29[https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\n2 [Install Google Credential Provider for Windows](https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)","status":"AVAILABLE"},{"created":"2025-10-24T01:04:22+00:00","modified":"2025-10-24T06:50:16+00:00","when":"2025-10-24T01:04:22+00:00","text":"**Summary**\nCustomers were unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nThe issue with Endpoint management for Windows enrollment has been resolved for all affected customers as of Thursday, October 23, 2025, 17:19 PDT.\nFrom preliminary analysis, the issue was caused by a browser compatibility problem with the deprecated IE11 used by Microsoft provided Windows Settings app. Our engineers resolved this by deploying a fix to the Google Identity servers to restore support for these requests.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers were not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) were not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone.","status":"AVAILABLE"},{"created":"2025-10-23T23:05:38+00:00","modified":"2025-10-24T05:40:57+00:00","when":"2025-10-23T23:05:38+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Friday, 2025-10-24 12:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T19:17:52+00:00","modified":"2025-10-24T05:41:12+00:00","when":"2025-10-21T19:17:52+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Thursday, 2025-10-23 17:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T03:58:16+00:00","modified":"2025-10-24T05:41:29+00:00","when":"2025-10-21T03:58:16+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Tuesday, 2025-10-21 11:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T02:59:05+00:00","modified":"2025-10-24T05:41:45+00:00","when":"2025-10-21T02:59:05+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Tuesday, 2025-10-21 01:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T01:39:50+00:00","modified":"2025-10-24T05:42:00+00:00","when":"2025-10-21T01:39:50+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Monday, 2025-10-20 20:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-20T21:41:22+00:00","modified":"2025-10-24T05:42:15+00:00","when":"2025-10-20T21:41:22+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Monday, 2025-10-20 18:30 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-20T19:50:06+00:00","modified":"2025-10-24T05:42:29+00:00","when":"2025-10-20T19:50:06+00:00","text":"**Summary**\nWe are investigating an issue impacting Endpoint management for Windows Devices\n**Description**\nWe are experiencing an issue with Endpoint management beginning on Wednesday, 2025-10-01 00:00 PDT. We detected the impact progressively increasing on Tuesday, 2025-10-14 10:23 PDT.\nThe Windows native settings app responsible for communicating with Google Identity and Endpoint management is using Internet Explorer 11 which has been deprecated by Microsoft. Google Identity servers are failing to serve requests for IE11 which is impacting device enrollment.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-10-20 15:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\u0026sjid=13524530415918932254-NC\u0026authuser=0)\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\u0026src=supportwidget0\u0026authuser=0)\n**Workaround**\nNone at this time","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-27T11:10:45+00:00","modified":"2025-10-27T11:10:46+00:00","when":"2025-10-27T11:10:45+00:00","text":"# Incident Report\n## Summary\nBeginning on 1 October 2025, Google Endpoint Management customers experienced issues enrolling new Windows devices using the native Windows Settings application or direct enrollment deeplinks. This incident started on 1 October 2025 at 00:00 US/Pacific and lasted for 22 days, 17 hours and 19 minutes, with resolution on 23 October 2025 at 17:19 US/Pacific. We sincerely apologize for any impact this disruption had on your device management workflows. We are taking measures to prevent a recurrence.\n## Root Cause\nThe root cause of this incident was a deprecated[1] browser compatibility issue. The device enrollment process initiated from the Windows Settings application relies on an embedded browser component. This component's behavior is based on the now deprecated Internet Explorer 11. Recent updates to Google's Identity servers, intended to enhance security and modernize the platform, inadvertently removed compatibility with the outdated protocols and behaviors exhibited by the Internet Explorer 11 engine. As a result, authentication requests from the Windows Settings enrollment flow could no longer be processed successfully by the updated Identity servers, causing the enrollment to fail.\n[1] Deprecated browser compatibility issue: https://learn.microsoft.com/en-us/lifecycle/products/internet-explorer-11\n## Remediation and Prevention\nThe issue was detected on Tuesday, 14 October 2025, at 10:23 US/Pacific, as the impact progressively increased, prompting a detailed investigation. Google engineers traced the failures back to the incompatibility between the Google Identity servers and the enrollment process embedded within the Windows Settings application, which uses a deprecated Internet Explorer 11 component.\nTo remediate the issue, Google engineers developed and deployed a targeted fix to the Google Identity servers. This update restored the servers' ability to handle authentication requests originating from the enrollment flow dependent on the Internet Explorer 11 technology. The rollout of this fix was completed on 23 October 2025 at 17:19 US/Pacific, fully restoring the ability to enroll new Windows devices via the Windows Settings application and deeplinks.\nGoogle is committed to preventing similar incidents in the future and is completing the following actions:\nWe are enhancing our testing procedures for Windows device enrollment flows.\nWe are exploring improvements in monitoring to detect issues on lower-traffic enrollment pathways more rapidly.\n## Detailed Description of Impact\nGoogle Endpoint Management - From 1 October 2025, 00:00 US/Pacific to 23 October 2025, 17:19 US/Pacific, customers were unable to enroll new Windows devices into Google Endpoint Management when using the built-in Windows Settings application (as described in “Enroll a device” in Windows device management) or an enrollment deeplink. The enrollment process would fail during the authentication phase due to the incompatibility.\nCustomers using the Google Credential Provider for Windows (GCPW) for device setup and login (as described in “Install Google Credential Provider for Windows”) were not affected by this incident and could continue to enroll and manage devices without issue. The impact was limited to the specific enrollment method relying on the Windows Settings interface.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"oDCiEGFp9igW5FDawdXx","service_name":"Endpoint Management","affected_products":[{"title":"Endpoint Management","id":"oDCiEGFp9igW5FDawdXx"}],"uri":"incidents/UYc9QNUSG1gniUUcDkDy"},{"id":"JiwWc1AHjeGuPxh6RUwJ","number":"518392520791595651","begin":"2025-10-09T12:25:00+00:00","created":"2025-10-09T15:09:28+00:00","end":"2025-10-09T13:20:00+00:00","modified":"2025-10-10T01:28:44+00:00","external_desc":"Some AppSheet users in Europe and America experienced increased latency, resulting in app synchronization failures and editor loading timeouts from Thursday, 2025-10-09 5:25 AM PDT to 2025-10-09 6:20 PDT\nWe will provide an update by Thursday, 2025-10-09 08:30 PDT with current details.","updates":[{"created":"2025-10-10T01:28:44+00:00","modified":"2025-10-10T01:28:45+00:00","when":"2025-10-10T01:28:44+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},{"created":"2025-10-09T17:52:40+00:00","modified":"2025-10-10T01:28:44+00:00","when":"2025-10-09T17:52:40+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article: https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 9 October 2025 at 00:10\n**Incident End:** 9 October 2025 at 06:20\n**Total Duration:** The incident occurred across two distinct impact windows:\n* Impact Window #1 (EMEA): 1 hour, 55 minutes\n* Impact Window #2 (EMEA \u0026 partial AMER): 55 minutes\n**Affected Services and Features:**\nAppSheet - App synchronization and Editor loading\n**Regions/Zones:**\n* EMEA\n* AMER (partial impact)\n**Description:**\nOn Thursday, 9 October 2025, some AppSheet users in the EMEA and AMER regions experienced elevated latency, resulting in app synchronization failures and editor loading timeouts. The cumulative duration of impact across both windows was 2 hours, 50 minutes.\nPreliminary investigation identified the issue was due to failures in autoscaling in 2 regions which caused servers to overload and cross-region failover did not occur. The issue was mitigated by rerouting traffic away from the impacted regions.\nGoogle will complete a full Incident Report in the following days that will provide a detailed root cause.\n**Customer Impact:**\nCustomers in the affected regions may have experienced high latency, resulting in errors when syncing data and loading AppSheet applications. The issue was initially observed in the EMEA region and was later observed in both EMEA and parts of the AMER region.\nThe timeline of the two distinct impact windows is as follows:\n* EMEA only: 9 October 2025, from 00:10 to 02:05 US/Pacific.\n* EMEA and partial AMER: 9 October 2025, from 05:25 to 06:20 US/Pacific.\n---","status":"AVAILABLE"},{"created":"2025-10-09T15:14:14+00:00","modified":"2025-10-09T17:52:40+00:00","when":"2025-10-09T15:14:14+00:00","text":"The issue with AppSheet in Europe and America is now resolved.\nFrom preliminary investigation the issue was due to resource contention that resulted in our servers not being able to startup during the duration of the issue. The issue was mitigated by rerouting traffic away from the impacted servers.","status":"AVAILABLE"},{"created":"2025-10-09T15:09:28+00:00","modified":"2025-10-09T15:14:14+00:00","when":"2025-10-09T15:09:28+00:00","text":"Some AppSheet users in Europe and America experienced increased latency, resulting in app synchronization failures and editor loading timeouts from Thursday, 2025-10-09 5:25 AM PDT to 2025-10-09 6:20 PDT\nWe will provide an update by Thursday, 2025-10-09 08:30 PDT with current details.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-10T01:28:44+00:00","modified":"2025-10-10T01:28:45+00:00","when":"2025-10-10T01:28:44+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/JiwWc1AHjeGuPxh6RUwJ"},{"id":"CBwZ6fszY7KFmncBCg2R","number":"4711405262404186636","begin":"2025-10-09T07:10:00+00:00","created":"2025-10-09T08:29:15+00:00","end":"2025-10-09T09:05:00+00:00","modified":"2025-10-10T01:30:46+00:00","external_desc":"**Summary**\nAppSheet users in Europe may encounter increased latency, resulting in app synchronization failures and editor loading timeouts.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-10-09 00:10 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-10-09 02:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nCustomers in the Europe may experience high latency, resulting in errors when syncing data and loading AppSheet applications\n**Workaround**\nNone at this time.","updates":[{"created":"2025-10-10T01:30:46+00:00","modified":"2025-10-10T01:30:46+00:00","when":"2025-10-10T01:30:46+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},{"created":"2025-10-09T17:53:30+00:00","modified":"2025-10-10T01:30:46+00:00","when":"2025-10-09T17:53:30+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article: https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 9 October 2025 at 00:10\n**Incident End:** 9 October 2025 at 06:20\n**Total Duration:** The incident occurred across two distinct impact windows:\n* Impact Window #1 (EMEA): 1 hour, 55 minutes\n* Impact Window #2 (EMEA \u0026 partial AMER): 55 minutes\n**Affected Services and Features:**\nAppSheet - App synchronization and Editor loading\n**Regions/Zones:**\n* EMEA\n* AMER (partial impact)\n**Description:**\nOn Thursday, 9 October 2025, some AppSheet users in the EMEA and AMER regions experienced elevated latency, resulting in app synchronization failures and editor loading timeouts. The cumulative duration of impact across both windows was 2 hours, 50 minutes.\nPreliminary investigation identified the issue was due to failures in autoscaling in 2 regions which caused servers to overload and cross-region failover did not occur. The issue was mitigated by rerouting traffic away from the impacted regions.\nGoogle will complete a full Incident Report in the following days that will provide a detailed root cause.\n**Customer Impact:**\nCustomers in the affected regions may have experienced high latency, resulting in errors when syncing data and loading AppSheet applications. The issue was initially observed in the EMEA region and was later observed in both EMEA and parts of the AMER region.\nThe timeline of the two distinct impact windows is as follows:\n* EMEA only: 9 October 2025, from 00:10 to 02:05 US/Pacific.\n* EMEA and partial AMER: 9 October 2025, from 05:25 to 06:20 US/Pacific.\n---","status":"AVAILABLE"},{"created":"2025-10-09T09:50:43+00:00","modified":"2025-10-09T17:53:30+00:00","when":"2025-10-09T09:50:43+00:00","text":"The problem with AppSheet has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-10-09T08:37:28+00:00","modified":"2025-10-09T09:50:43+00:00","when":"2025-10-09T08:37:28+00:00","text":"**Summary**\nAppSheet users in Europe may encounter increased latency, resulting in app synchronization failures and editor loading timeouts.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Thursday, 2025-10-09 03:00 PDT.\n**Customer Symptoms**\nCustomers in the Europe may experience high latency, resulting in errors when syncing data and loading AppSheet applications\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-09T08:29:15+00:00","modified":"2025-10-09T08:37:28+00:00","when":"2025-10-09T08:29:15+00:00","text":"**Summary**\nAppSheet users in Europe may encounter increased latency, resulting in app synchronization failures and editor loading timeouts.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-10-09 00:10 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-10-09 02:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nCustomers in the Europe may experience high latency, resulting in errors when syncing data and loading AppSheet applications\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-10T01:30:46+00:00","modified":"2025-10-10T01:30:46+00:00","when":"2025-10-10T01:30:46+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/CBwZ6fszY7KFmncBCg2R"},{"id":"xZEAwY6ckkm8kkjRa2Kp","number":"4868498765151813095","begin":"2025-10-03T17:16:00+00:00","created":"2025-10-03T20:07:52+00:00","end":"2025-10-03T22:00:00+00:00","modified":"2025-10-03T22:57:02+00:00","external_desc":"**Summary:**\nUsers using non-Gmail accounts in the Gmail app may encounter an error message requiring additional steps.\n**Description:**\nOur engineering team has identified the root cause and mitigation work is currently underway.\nThe mitigation is expected to complete by Friday, 2025-10-03 15:00 PDT\nWe will provide an update by Friday, 2025-10-03 15:30 PDT with details.\n**Diagnosis / Customer Symptoms:**\nImpacted users may see an error message when opening the Gmail app with a non-Gmail account (e.g. Yahoo, Hotmail, etc) saying they need to complete additional steps before using the Gmail app. After that, their inbox will have a banner saying their account is offline.\nThe app can be used normally even while the \"offline\" banner is visible.\nThis is only impacting mobile Android users at the moment, both consumers and enterprise users.\nDesktop users are not impacted.\n**Workaround:**\nUsers can keep using the app as usual, or they can use alternative clients (e.g. desktop).","updates":[{"created":"2025-10-03T22:57:02+00:00","modified":"2025-10-03T22:57:02+00:00","when":"2025-10-03T22:57:02+00:00","text":"The issue with Gmail has been resolved for all affected users as of Friday, 2025-10-03 15:00 PDT.\nFrom preliminary analysis, the issue was triggered by a recent change that was deployed as part of some improvements to Android authentication. Once the underlying cause was identified, our engineers mitigated the issue by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-10-03T20:07:52+00:00","modified":"2025-10-03T22:57:02+00:00","when":"2025-10-03T20:07:52+00:00","text":"**Summary:**\nUsers using non-Gmail accounts in the Gmail app may encounter an error message requiring additional steps.\n**Description:**\nOur engineering team has identified the root cause and mitigation work is currently underway.\nThe mitigation is expected to complete by Friday, 2025-10-03 15:00 PDT\nWe will provide an update by Friday, 2025-10-03 15:30 PDT with details.\n**Diagnosis / Customer Symptoms:**\nImpacted users may see an error message when opening the Gmail app with a non-Gmail account (e.g. Yahoo, Hotmail, etc) saying they need to complete additional steps before using the Gmail app. After that, their inbox will have a banner saying their account is offline.\nThe app can be used normally even while the \"offline\" banner is visible.\nThis is only impacting mobile Android users at the moment, both consumers and enterprise users.\nDesktop users are not impacted.\n**Workaround:**\nUsers can keep using the app as usual, or they can use alternative clients (e.g. desktop).","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-03T22:57:02+00:00","modified":"2025-10-03T22:57:02+00:00","when":"2025-10-03T22:57:02+00:00","text":"The issue with Gmail has been resolved for all affected users as of Friday, 2025-10-03 15:00 PDT.\nFrom preliminary analysis, the issue was triggered by a recent change that was deployed as part of some improvements to Android authentication. Once the underlying cause was identified, our engineers mitigated the issue by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/xZEAwY6ckkm8kkjRa2Kp"},{"id":"6GdahVtGTryxTdh56iLP","number":"3413083329319526622","begin":"2025-09-29T17:53:00+00:00","created":"2025-09-29T18:58:21+00:00","end":"2025-09-29T18:42:00+00:00","modified":"2025-10-02T20:17:06+00:00","external_desc":"We're investigating reports of an issue with Gemini. We will provide more information shortly.","updates":[{"created":"2025-10-02T20:17:06+00:00","modified":"2025-10-02T20:17:06+00:00","when":"2025-10-02T20:17:06+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},{"created":"2025-09-30T05:12:51+00:00","modified":"2025-10-02T20:17:06+00:00","when":"2025-09-30T05:12:51+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 29 September, 2025 10:53\n**Incident End:** 29 September, 2025 11:42\n**Duration:** 49 minutes\n**GWS Affected Services and Features:**\nWorkspace Gemini and NotebookLM\n**Regions/Zones:** Global\n**Description:**\nVarious AI products experienced elevated 500 (internal) errors due to the rollout of a faulty change to production. Google will complete a full IR in the following days that will provide a full root cause.\n**Customer Impact:**\nCustomers affected by this issue may have observed up to a 100% of requests failing with '500' errors during the impacted time, between 10:53 and 11:42 US/Pacific on 29 September 2025.\n**Additional details:**\nA faulty change was rolled out to the control plane of the canonical Machine Learning (ML) serving stack in one cluster. Traffic routers that received their configuration from the faulty cluster assumed that some models were deleted and returned errors to the clients. Due to the error, the faulty control plane cluster had a significantly lower load, which resulted in more routers getting their configuration from the faulty cluster. This exacerbated the impact of the change.\nThe bad change was rolled back and all the errors returned by ML serving were reduced to pre-fault levels, mitigating customer impact.","status":"AVAILABLE"},{"created":"2025-09-29T19:48:03+00:00","modified":"2025-09-30T05:12:51+00:00","when":"2025-09-29T19:48:03+00:00","text":"**Summary:**\nWorkSpace Gemini customers may experience elevated errors for online users\n**Description:**\nThe issue with Workspace Gemini has been mitigated as of Monday, 2025-09-29 11:42 PDT.\nBased on preliminary investigation, the root cause of the issue was due to a recent change, which has been rolled back.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"SERVICE_INFORMATION"},{"created":"2025-09-29T19:37:23+00:00","modified":"2025-09-29T19:48:03+00:00","when":"2025-09-29T19:37:23+00:00","text":"**Summary:**\nWorkSpace Gemini customers may experience elevated errors for online users\n**Description:**\nOur engineering team has identified the cause of the issue and a mitigation has been put in place. We are currently monitoring and believe we are seeing full recovery.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"SERVICE_INFORMATION"},{"created":"2025-09-29T18:58:21+00:00","modified":"2025-09-29T19:37:23+00:00","when":"2025-09-29T18:58:21+00:00","text":"We're investigating reports of an issue with Gemini. We will provide more information shortly. Some users would observe elevated errors from Gemini for Workspace.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-02T20:17:06+00:00","modified":"2025-10-02T20:17:06+00:00","when":"2025-10-02T20:17:06+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"npdyhgECDJ6tB66MxXyo","service_name":"Gemini","affected_products":[{"title":"Gemini","id":"npdyhgECDJ6tB66MxXyo"}],"uri":"incidents/6GdahVtGTryxTdh56iLP"},{"id":"JWwiKhWRMBJB6Pi9gdws","number":"5905233132377111505","begin":"2025-09-29T17:53:00+00:00","created":"2025-09-29T18:57:13+00:00","end":"2025-09-29T18:42:00+00:00","modified":"2025-10-02T20:17:40+00:00","external_desc":"We're investigating reports of an issue with NotebookLM. We will provide more information shortly.","updates":[{"created":"2025-10-02T20:17:40+00:00","modified":"2025-10-02T20:17:40+00:00","when":"2025-10-02T20:17:40+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},{"created":"2025-09-30T05:15:40+00:00","modified":"2025-10-02T20:17:40+00:00","when":"2025-09-30T05:15:40+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 29 September, 2025 10:53\n**Incident End:** 29 September, 2025 11:42\n**Duration:** 49 minutes\n**GWS Affected Services and Features:**\nWorkspace Gemini and NotebookLM\n**Regions/Zones:** Global\n**Description:**\nVarious AI products experienced elevated 500 (internal) errors due to the rollout of a faulty change to production. Google will complete a full IR in the following days that will provide a full root cause.\n**Customer Impact:**\nCustomers affected by this issue may have observed up to a 100% of requests failing with '500' errors during the impacted time, between 10:53 and 11:42 US/Pacific on 29 September 2025.\n**Additional details:**\nA faulty change was rolled out to the control plane of the canonical Machine Learning (ML) serving stack in one cluster. Traffic routers that received their configuration from the faulty cluster assumed that some models were deleted and returned errors to the clients. Due to the error, the faulty control plane cluster had a significantly lower load, which resulted in more routers getting their configuration from the faulty cluster. This exacerbated the impact of the change.\nThe bad change was rolled back and all the errors returned by ML serving were reduced to pre-fault levels, mitigating customer impact.","status":"AVAILABLE"},{"created":"2025-09-29T19:56:21+00:00","modified":"2025-09-30T05:15:40+00:00","when":"2025-09-29T19:56:21+00:00","text":"**Summary:**\nWorkSpace NotebookLM customers may experience elevated errors for online users\n**Description:**\nThe issue with Workspace NotebookLM has been mitigated as of Monday, 2025-09-29 11:49 PDT.\nBased on preliminary investigation, the root cause of the issue was due to a recent change, which has been rolled back.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"AVAILABLE"},{"created":"2025-09-29T19:39:58+00:00","modified":"2025-09-29T19:56:21+00:00","when":"2025-09-29T19:39:58+00:00","text":"**Summary:**\nNotebookLM customers may experience elevated errors for online users\n**Description:**\nOur engineering team has identified the cause of the issue and a mitigation has been put in place. We are currently monitoring and believe we are seeing full recovery.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"SERVICE_INFORMATION"},{"created":"2025-09-29T18:57:13+00:00","modified":"2025-09-29T19:39:58+00:00","when":"2025-09-29T18:57:13+00:00","text":"We're investigating reports of an issue with NotebookLM. We will provide more information shortly. Some users would observe elevated errors from Gemini for Workspace.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-02T20:17:40+00:00","modified":"2025-10-02T20:17:40+00:00","when":"2025-10-02T20:17:40+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sqTm5ZmzCmb66kvyzcNS","service_name":"NotebookLM","affected_products":[{"title":"NotebookLM","id":"sqTm5ZmzCmb66kvyzcNS"}],"uri":"incidents/JWwiKhWRMBJB6Pi9gdws"},{"id":"Cy8B1om17oecUkKm3G9Q","number":"7174538375727097808","begin":"2025-09-26T15:49:00+00:00","created":"2025-09-26T16:52:53+00:00","end":"2025-09-26T16:47:00+00:00","modified":"2025-10-02T02:01:02+00:00","external_desc":"**Summary:**\nGoogle Meet may be unavailable for users globally.\n**Description:**\nWe are experiencing an issue with Google Meet beginning on Friday, 2025-09-26 08:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 11:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nUsers are unable to join calls on Google Meet.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-10-02T00:08:13+00:00","modified":"2025-10-02T02:01:02+00:00","when":"2025-10-02T00:08:13+00:00","text":"# Incident Report\n## Summary\nOn Friday, 26 September 2025 at 08:40 US/Pacific, Google Meet users experienced elevated error rates globally for 1 hour, 1 minute. To our Google Meet customers who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe root cause of the incident was a flaw in the error-handling logic within the experimentation module's configuration evaluation. This flaw created the conditions for resource contention, where too many processes attempted to access the same system resource at once.\nThe trigger was a specific change to the Meet experimentation configuration file, which caused an elevated error level during evaluation and led to the contention in the Meet backend. Our monitoring systems detected the issue right away, and engineers quickly identified the change as the source. They reverted the update, which resolved the issue.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue through the internal alerts at 08:49 US/Pacific on 26 September, 2025 and immediately started an investigation. They identified several recent changes as potential causes, and began systematically rolling back those changes.\nAt 09:25 US/Pacific, engineers reverted the Meet API Server binary to a previous version.\nAt 09:41 US/Pacific, engineers reverted and paused subsequent Meet backend API server configuration rollouts across all Google Meet production clusters thus mitigating the issue.\nAt 11:52 US/Pacific, engineers reverted the change that caused lock contention.\nWe sincerely apologize to our customers who were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking the following steps to ensure this does not happen again. The Google Meet Engineering team is accelerating the deprecation of legacy code that caused lock contention.\n## Detailed Description of Impact\nOn Friday, 26 September 2025, from 08:40 to 09:41 US/Pacific, users were either unable to connect to Google Meet or experienced problems with establishing meetings. Users already in Google Meet sessions were likely not impacted. The issue affected up to 20 percent of Google Meet users globally.","status":"AVAILABLE"},{"created":"2025-09-26T19:08:58+00:00","modified":"2025-10-02T00:08:13+00:00","when":"2025-09-26T19:08:58+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 26 September, 2025 08:40\n**Incident End:** 26 September, 2025 09:47\n**Duration:** 1 hours, 7 minutes\n**Affected Services and Features:**\nGoogle Meet\n**Regions/Zones:** Global\n**Description:**\nGoogle Meet users experienced elevated error rates globally for a duration of 1 hour, 7 minutes. The preliminary root cause was a recent rollout to a backend API server, which was mitigated after the rollback completed.\nGoogle will complete a full Incident Report in the following days that will provide a full root cause.\n**Customer Impact:**\nAffected users experienced 504 errors when attempting to access Google Meet.\n---","status":"AVAILABLE"},{"created":"2025-09-26T17:17:51+00:00","modified":"2025-09-26T19:08:58+00:00","when":"2025-09-26T17:17:51+00:00","text":"**Summary:**\nSome users experienced errors when trying to access Google Meet.\n**Description:**\nThe issue with Google Meet has been resolved for all affected users as of Friday, 2025-09-26 09:47 PDT.\nA root cause analysis is in progress. Google will be publishing a preliminary Incident Report shortly and a full Incident Report in the coming days.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms:**\nImpacted users experienced errors when attempting to access Google Meet.","status":"AVAILABLE"},{"created":"2025-09-26T16:52:53+00:00","modified":"2025-09-26T17:17:51+00:00","when":"2025-09-26T16:52:53+00:00","text":"**Summary:**\nGoogle Meet may be unavailable for users globally.\n**Description:**\nWe are experiencing an issue with Google Meet beginning on Friday, 2025-09-26 08:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 11:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nUsers are unable to join calls on Google Meet.\n**Workaround:**\nNone at this time.","status":"SERVICE_OUTAGE"}],"most_recent_update":{"created":"2025-10-02T00:08:13+00:00","modified":"2025-10-02T02:01:02+00:00","when":"2025-10-02T00:08:13+00:00","text":"# Incident Report\n## Summary\nOn Friday, 26 September 2025 at 08:40 US/Pacific, Google Meet users experienced elevated error rates globally for 1 hour, 1 minute. To our Google Meet customers who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe root cause of the incident was a flaw in the error-handling logic within the experimentation module's configuration evaluation. This flaw created the conditions for resource contention, where too many processes attempted to access the same system resource at once.\nThe trigger was a specific change to the Meet experimentation configuration file, which caused an elevated error level during evaluation and led to the contention in the Meet backend. Our monitoring systems detected the issue right away, and engineers quickly identified the change as the source. They reverted the update, which resolved the issue.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue through the internal alerts at 08:49 US/Pacific on 26 September, 2025 and immediately started an investigation. They identified several recent changes as potential causes, and began systematically rolling back those changes.\nAt 09:25 US/Pacific, engineers reverted the Meet API Server binary to a previous version.\nAt 09:41 US/Pacific, engineers reverted and paused subsequent Meet backend API server configuration rollouts across all Google Meet production clusters thus mitigating the issue.\nAt 11:52 US/Pacific, engineers reverted the change that caused lock contention.\nWe sincerely apologize to our customers who were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking the following steps to ensure this does not happen again. The Google Meet Engineering team is accelerating the deprecation of legacy code that caused lock contention.\n## Detailed Description of Impact\nOn Friday, 26 September 2025, from 08:40 to 09:41 US/Pacific, users were either unable to connect to Google Meet or experienced problems with establishing meetings. Users already in Google Meet sessions were likely not impacted. The issue affected up to 20 percent of Google Meet users globally.","status":"AVAILABLE"},"status_impact":"SERVICE_OUTAGE","severity":"high","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/Cy8B1om17oecUkKm3G9Q"},{"id":"rQD4joDQXiqfecRaVPRR","number":"9848531984137254167","begin":"2025-09-26T09:02:00+00:00","created":"2025-09-26T12:02:29+00:00","end":"2025-09-26T15:02:00+00:00","modified":"2025-10-06T15:01:54+00:00","external_desc":"We're investigating reports of an issue with Admin Console. We will provide more information shortly.","updates":[{"created":"2025-10-04T00:56:40+00:00","modified":"2025-10-06T11:25:00+00:00","when":"2025-10-04T00:56:40+00:00","text":"# Incident Report\n**Incident Start:** 26 September 2025 02:02\n**Incident End:** 26 September 2025 08:02\n**Duration:** 6 hours\n## Summary\nStarting on Friday, 26 September 2025 02:02 US/Pacific, a power equipment failure in zone us-central1-a caused service disruptions. Google Workspace Admin Console experienced elevated latency and error rates. Service was fully restored by 08:02 US/Pacific.\n## Root Cause\nThe incident was triggered by a power equipment failure at one of the data centers in us-central1-a at 02:02 US/Pacific. This failure resulted in a power disruption to a subset of storage servers within the zone, leading to the observed impact across the listed services.\n## Detailed Description of Impact\n#### Admin Console (Google Workspace)\nA majority of Admin-exposed services, such as the Security Investigation Tool (SIT), Email Log Search (ELS), and Admin Quarantine, experienced issues where reporting pages appeared blank, charts failed to load, or a buffering circle was continuously displayed. Some services also returned ‘502’ (Bad Gateway) errors. The services were fully mitigated by 08:02 US/Pacific. The delayed recovery was caused by a dependency on Google BigQuery, which was also impacted by the incident.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident from an internal monitoring alert at 02:13 US/Pacific on 26 September 2025 and addressed the incident by transferring the affected servers to alternate power equipment, which was completed by 02:35 US/Pacific.\nA majority of the impacted services recovered quickly upon the resumption of power. However, several products had extended time to recover due to their dependency on the impacted storage system.\nTo protect the data during a power event, a safety feature was auto activated. This feature temporarily placed affected storage servers into a protective, read-only mode to prevent any data from being written that might not be saved correctly in case of another power loss.\nWhile this safety feature worked as intended to ensure data durability, it had an unintended side effect. Our underlying file system, which orchestrates data storage, needs to perform very small, internal \"write\" operations to coordinate its tasks, even when you are only reading data. Because a large number of storage servers were in the read-only state, the file system was unable to perform these essential internal writes. This caused parts of the file system to become temporarily unavailable, which in turn impacted services like Persistent Disk that rely on it.\nWe sincerely apologize to our customers whose services were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking steps to ensure this does not happen again.\n* **Speeding storage system recovery:** We are reverting the recent change, to the previous behavior. We will improve testing of this feature to ensure it does not introduce availability issues after power loss events before reintroducing it at a later date.\n* **Enhancing our monitoring:** We are building a new dashboard to give us a real-time, comprehensive view of the power and battery status of all our storage servers. This will allow us to detect and respond to potential power-related issues much more quickly.\n* **Strengthening data-protection protocols:** We are creating clearer guidelines for our engineers on when it is appropriate to override automated data-protection features. This will help ensure that we always prioritize the safety and durability of your data.\n* **Rigorously testing our safety features:** We are implementing more thorough fault testing to confirm that our data-protection mechanisms work exactly as expected, especially after they have been activated and then reset.\n* **Improving dependent system recovery:** We are implementing changes in BigQuery to improve the time to recovery of the BigQuery processing for recently ingested data after a partial or full cluster level storage system outage.","status":"AVAILABLE"},{"created":"2025-09-27T01:12:11+00:00","modified":"2025-10-06T15:01:54+00:00","when":"2025-09-27T01:12:11+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 26 September 2025 02:02\n**Incident End:** 26 September 2025 08:00\n**Duration:** 5 hours, 58 minutes\n**Affected Services and Features:**\nAdmin Console\n**Description:**\nStarting at 02:02 on Friday, 26 September 2025, multiple Google services in a specific zone began experiencing elevated latency and error rates. A power equipment failure at one of the datacenters in the zone resulted in power disruption for a subset of storage servers.\nTo restore services, the underlying power issue was immediately addressed by transferring the affected servers to an alternate power equipment by 02:35.\nA majority of the impacted services were recovered quickly upon resumption of power. Due to manual intervention required to restore a small subset of the storage servers, a few services continued to experience partial unavailability and were fully recovered by 08:00.\nGoogle will complete a full Incident Report in the following days that will provide a full root cause.\n**Customer Impact:**\nWorkspace Admin Console users may have observed ‘502’ errors (Bad Gateway) or continuous loading.","status":"AVAILABLE"},{"created":"2025-09-26T15:49:11+00:00","modified":"2025-09-27T01:12:11+00:00","when":"2025-09-26T15:49:11+00:00","text":"**Description :**\nThe issue with Admin Console has been resolved for all affected users as of Friday, 2025-09-26 07:56 PT.\nThe issue was triggered by a backend storage service issue. We are investigating the full root cause, and we will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T15:01:44+00:00","modified":"2025-09-26T15:49:11+00:00","when":"2025-09-26T15:01:44+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 09:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T13:53:11+00:00","modified":"2025-09-26T15:01:44+00:00","when":"2025-09-26T13:53:11+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 08:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T13:25:10+00:00","modified":"2025-09-26T13:53:11+00:00","when":"2025-09-26T13:25:10+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 07:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:51:28+00:00","modified":"2025-09-26T13:25:10+00:00","when":"2025-09-26T12:51:28+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 06:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:24:50+00:00","modified":"2025-09-26T12:51:28+00:00","when":"2025-09-26T12:24:50+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 06:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:07:15+00:00","modified":"2025-09-26T12:24:50+00:00","when":"2025-09-26T12:07:15+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 05:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:02:29+00:00","modified":"2025-09-26T12:07:15+00:00","when":"2025-09-26T12:02:29+00:00","text":"We're investigating reports of an issue with Admin Console. We will provide more information shortly.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-04T00:56:40+00:00","modified":"2025-10-06T11:25:00+00:00","when":"2025-10-04T00:56:40+00:00","text":"# Incident Report\n**Incident Start:** 26 September 2025 02:02\n**Incident End:** 26 September 2025 08:02\n**Duration:** 6 hours\n## Summary\nStarting on Friday, 26 September 2025 02:02 US/Pacific, a power equipment failure in zone us-central1-a caused service disruptions. Google Workspace Admin Console experienced elevated latency and error rates. Service was fully restored by 08:02 US/Pacific.\n## Root Cause\nThe incident was triggered by a power equipment failure at one of the data centers in us-central1-a at 02:02 US/Pacific. This failure resulted in a power disruption to a subset of storage servers within the zone, leading to the observed impact across the listed services.\n## Detailed Description of Impact\n#### Admin Console (Google Workspace)\nA majority of Admin-exposed services, such as the Security Investigation Tool (SIT), Email Log Search (ELS), and Admin Quarantine, experienced issues where reporting pages appeared blank, charts failed to load, or a buffering circle was continuously displayed. Some services also returned ‘502’ (Bad Gateway) errors. The services were fully mitigated by 08:02 US/Pacific. The delayed recovery was caused by a dependency on Google BigQuery, which was also impacted by the incident.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident from an internal monitoring alert at 02:13 US/Pacific on 26 September 2025 and addressed the incident by transferring the affected servers to alternate power equipment, which was completed by 02:35 US/Pacific.\nA majority of the impacted services recovered quickly upon the resumption of power. However, several products had extended time to recover due to their dependency on the impacted storage system.\nTo protect the data during a power event, a safety feature was auto activated. This feature temporarily placed affected storage servers into a protective, read-only mode to prevent any data from being written that might not be saved correctly in case of another power loss.\nWhile this safety feature worked as intended to ensure data durability, it had an unintended side effect. Our underlying file system, which orchestrates data storage, needs to perform very small, internal \"write\" operations to coordinate its tasks, even when you are only reading data. Because a large number of storage servers were in the read-only state, the file system was unable to perform these essential internal writes. This caused parts of the file system to become temporarily unavailable, which in turn impacted services like Persistent Disk that rely on it.\nWe sincerely apologize to our customers whose services were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking steps to ensure this does not happen again.\n* **Speeding storage system recovery:** We are reverting the recent change, to the previous behavior. We will improve testing of this feature to ensure it does not introduce availability issues after power loss events before reintroducing it at a later date.\n* **Enhancing our monitoring:** We are building a new dashboard to give us a real-time, comprehensive view of the power and battery status of all our storage servers. This will allow us to detect and respond to potential power-related issues much more quickly.\n* **Strengthening data-protection protocols:** We are creating clearer guidelines for our engineers on when it is appropriate to override automated data-protection features. This will help ensure that we always prioritize the safety and durability of your data.\n* **Rigorously testing our safety features:** We are implementing more thorough fault testing to confirm that our data-protection mechanisms work exactly as expected, especially after they have been activated and then reset.\n* **Improving dependent system recovery:** We are implementing changes in BigQuery to improve the time to recovery of the BigQuery processing for recently ingested data after a partial or full cluster level storage system outage.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/rQD4joDQXiqfecRaVPRR"},{"id":"mLrAjo9VbGGzQJRt7TQM","number":"6479781004903047668","begin":"2025-09-24T18:51:00+00:00","created":"2025-09-25T00:43:43+00:00","end":"2025-09-25T08:00:00+00:00","modified":"2025-09-25T09:26:40+00:00","external_desc":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-09-24 19:30 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","updates":[{"created":"2025-09-25T09:26:40+00:00","modified":"2025-09-25T09:26:40+00:00","when":"2025-09-25T09:26:40+00:00","text":"The problem with Admin Console has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-09-25T07:43:35+00:00","modified":"2025-09-25T09:26:40+00:00","when":"2025-09-25T07:43:35+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Thursday, 2025-09-25 05:00 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-25T03:05:27+00:00","modified":"2025-09-25T07:43:35+00:00","when":"2025-09-25T03:05:27+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Thursday, 2025-09-25 01:00 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-25T02:18:46+00:00","modified":"2025-09-25T03:05:27+00:00","when":"2025-09-25T02:18:46+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-09-24 21:00 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-25T00:43:43+00:00","modified":"2025-09-25T02:18:46+00:00","when":"2025-09-25T00:43:43+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-09-24 19:30 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-25T09:26:40+00:00","modified":"2025-09-25T09:26:40+00:00","when":"2025-09-25T09:26:40+00:00","text":"The problem with Admin Console has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/mLrAjo9VbGGzQJRt7TQM"},{"id":"5V5yK8N8heBKnmdqS1eW","number":"1782113629924000856","begin":"2025-09-18T14:22:00+00:00","created":"2025-09-18T16:03:13+00:00","end":"2025-09-18T15:35:00+00:00","modified":"2025-09-24T17:30:38+00:00","external_desc":"**Summary:**\nWe have mitigated an issue where some customers experienced login issues to Google services.\n**Description:**\nBetween 07:22 and 08:30 PDT, customers experienced login issues to Google services.\nOur engineering team has identified the cause and applied appropriate mitigations by 08:30 PDT. Customers should now be able to login to the Google services.\nWe are closely monitoring the environment for stability.\nWe will provide an update by Thursday, 2025-09-18 09:20 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers experienced login issues to Google services.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-09-24T17:27:52+00:00","modified":"2025-09-24T17:30:38+00:00","when":"2025-09-24T17:27:52+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 18 September 2025, between 07:22 and 08:35 US/Pacific, Google Workspace products experienced elevated login failures globally for a duration of 1 hour 13 minutes. Some users may have been unable to login to various Google Workspace services during that time.\nThis is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA shift in traffic led to a resource contention issue within Google's authentication system. The system's capacity for handling concurrent requests was exhausted in several east coast regions, creating a bottleneck that blocked new login requests. This resulted in elevated login failures and errors for users across various Google services.\n## Remediation and Prevention\nGoogle engineers were alerted at 07:31 US/Pacific via internal monitoring. After identifying the bottleneck, engineers made configuration changes to increase the replica count of the affected frontend service component by 50%. This additional capacity relieved the resource contention, and normal service was fully restored by 08:35 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- We will tune the current authentication frontend's deployment parameters to ensure ongoing system stability\n- Longer term, we plan to eliminate this class of resource contention by migrating to a new architecture.","status":"AVAILABLE"},{"created":"2025-09-18T23:44:08+00:00","modified":"2025-09-24T17:27:52+00:00","when":"2025-09-18T23:44:08+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 18 September 2025, 07:20\n**Incident End:** 18 September 2025, 08:30\n**Duration:** 1 hour, 10 minutes\n**Affected Services and Features:**\nAll Google Workspace products\n**Description:**\nMultiple customers experienced elevated login failures within various Workspace products. From preliminary analysis, the incident was suspected to be caused by a resource contention issue in Google’s internal authentication system.\nGoogle will complete a full incident report in the following days that will provide a full root cause.\n**Customer Impact:**\nMultiple users may have been unable to login to various Google services during the impacted time.","status":"AVAILABLE"},{"created":"2025-09-18T16:53:36+00:00","modified":"2025-09-18T23:44:08+00:00","when":"2025-09-18T16:53:36+00:00","text":"**Summary:**\nWe have mitigated an issue where some customers experienced login issues to Google services.\n**Description:**\nThe issue with Google services has been resolved for all affected customers as of Thursday, 2025-09-18 08:30 US/Pacific.\nFrom preliminary analysis, we have identified that the issue is caused by a resource contention issue. Our engineers have mitigated the issue by adding in additional resources.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nSome customers experienced login issues to Google services.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-09-18T16:03:13+00:00","modified":"2025-09-18T16:53:36+00:00","when":"2025-09-18T16:03:13+00:00","text":"**Summary:**\nWe have mitigated an issue where some customers experienced login issues to Google services.\n**Description:**\nBetween 07:22 and 08:30 PDT, customers experienced login issues to Google services.\nOur engineering team has identified the cause and applied appropriate mitigations by 08:30 PDT. Customers should now be able to login to the Google services.\nWe are closely monitoring the environment for stability.\nWe will provide an update by Thursday, 2025-09-18 09:20 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers experienced login issues to Google services.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-09-24T17:27:52+00:00","modified":"2025-09-24T17:30:38+00:00","when":"2025-09-24T17:27:52+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 18 September 2025, between 07:22 and 08:35 US/Pacific, Google Workspace products experienced elevated login failures globally for a duration of 1 hour 13 minutes. Some users may have been unable to login to various Google Workspace services during that time.\nThis is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA shift in traffic led to a resource contention issue within Google's authentication system. The system's capacity for handling concurrent requests was exhausted in several east coast regions, creating a bottleneck that blocked new login requests. This resulted in elevated login failures and errors for users across various Google services.\n## Remediation and Prevention\nGoogle engineers were alerted at 07:31 US/Pacific via internal monitoring. After identifying the bottleneck, engineers made configuration changes to increase the replica count of the affected frontend service component by 50%. This additional capacity relieved the resource contention, and normal service was fully restored by 08:35 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- We will tune the current authentication frontend's deployment parameters to ensure ongoing system stability\n- Longer term, we plan to eliminate this class of resource contention by migrating to a new architecture.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"},{"title":"Chrome Browser","id":"sfPUGuj9d6b63iYijuYV"},{"title":"ChromeOS","id":"RjKbsvuGeE7M1pvBHmvX"},{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/5V5yK8N8heBKnmdqS1eW"},{"id":"mYvDLhU69KiPAdE967TZ","number":"4739091364163717407","begin":"2025-09-11T17:04:00+00:00","created":"2025-09-11T20:46:35+00:00","end":"2025-09-11T21:27:00+00:00","modified":"2025-09-12T02:58:16+00:00","external_desc":"**Summary:**\nWe are experiencing an issue with NotebookLM\n**Description:**\nWe are experiencing an issue with AudioOverview, VideoOverview, NAPPS (Notebook Apps) for NotebookLM users, beginning on Thursday, 2025-09-11 10:04 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have a full mitigation ETA at this time.\nWe will provide an update by Thursday, 2025-09-11 15:00 PDT with current details.\n**Symptoms:**\nNotebookLM users may see failures to generate:\n- Audio Overviews\n- Video Overviews\n- Reports\n- Flashcards\n- Quizzes\nFeatures launched earlier this week have been rolled back for now.\n(Quizzes, Flashcards, Reports) (https://blog.google/technology/google-labs/notebooklm-student-features/)\nAny previously generated artifacts are accessible. All other features and CUJs are believed to be working. E.g. text Q\u0026A chat, adding sources should work\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-09-11T22:49:59+00:00","modified":"2025-09-12T02:58:16+00:00","when":"2025-09-11T22:49:59+00:00","text":"**Summary:**\nNotebookLM users may have encountered failures when generating some features.\n**Description:**\nThe issue with Audio Overviews and Video Overviews has been resolved for all affected users as of Thursday, 2025-09-11 14:27 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nNotebookLM users may have encountered failures to generate:\n* Audio Overviews\n* Video Overviews\n* Reports\n* Flashcards\n* Quizzes\nFeatures launched earlier this week (Quizzes, Flashcards, Reports) have been temporarily rolled back as part of the mitigation. (https://blog.google/technology/google-labs/notebooklm-student-features/)\nAny previously generated artifacts were accessible. All other features and CUJs were unaffected by this issue; e.g. Text Q\u0026A chat, adding sources.","status":"AVAILABLE"},{"created":"2025-09-11T20:46:35+00:00","modified":"2025-09-11T22:49:59+00:00","when":"2025-09-11T20:46:35+00:00","text":"**Summary:**\nWe are experiencing an issue with NotebookLM\n**Description:**\nWe are experiencing an issue with AudioOverview, VideoOverview, NAPPS (Notebook Apps) for NotebookLM users, beginning on Thursday, 2025-09-11 10:04 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have a full mitigation ETA at this time.\nWe will provide an update by Thursday, 2025-09-11 15:00 PDT with current details.\n**Symptoms:**\nNotebookLM users may see failures to generate:\n- Audio Overviews\n- Video Overviews\n- Reports\n- Flashcards\n- Quizzes\nFeatures launched earlier this week have been rolled back for now.\n(Quizzes, Flashcards, Reports) (https://blog.google/technology/google-labs/notebooklm-student-features/)\nAny previously generated artifacts are accessible. All other features and CUJs are believed to be working. E.g. text Q\u0026A chat, adding sources should work\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-09-11T22:49:59+00:00","modified":"2025-09-12T02:58:16+00:00","when":"2025-09-11T22:49:59+00:00","text":"**Summary:**\nNotebookLM users may have encountered failures when generating some features.\n**Description:**\nThe issue with Audio Overviews and Video Overviews has been resolved for all affected users as of Thursday, 2025-09-11 14:27 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nNotebookLM users may have encountered failures to generate:\n* Audio Overviews\n* Video Overviews\n* Reports\n* Flashcards\n* Quizzes\nFeatures launched earlier this week (Quizzes, Flashcards, Reports) have been temporarily rolled back as part of the mitigation. (https://blog.google/technology/google-labs/notebooklm-student-features/)\nAny previously generated artifacts were accessible. All other features and CUJs were unaffected by this issue; e.g. Text Q\u0026A chat, adding sources.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sqTm5ZmzCmb66kvyzcNS","service_name":"NotebookLM","affected_products":[{"title":"NotebookLM","id":"sqTm5ZmzCmb66kvyzcNS"}],"uri":"incidents/mYvDLhU69KiPAdE967TZ"},{"id":"muX1XjAhRG1krLmwbQsW","number":"5192794371337674379","begin":"2025-09-08T17:25:00+00:00","created":"2025-09-08T17:49:31+00:00","end":"2025-09-08T18:17:00+00:00","modified":"2025-09-22T13:48:55+00:00","external_desc":"**Summary:**\nWe are investigating reports of Google Meet service issues.\n**Description:**\nWe are investigating reports of Google Meet degradation\nWe will provide more information by Monday, 2025-09-08 11:15 PDT.\n**Symptoms:**\nCustomers may be experiencing elevated errors when joining meet sessions.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-09-18T17:00:33+00:00","modified":"2025-09-22T13:48:55+00:00","when":"2025-09-18T17:00:33+00:00","text":"# Incident Report\n## Summary\nOn 8 September 2025, beginning at 00:00 US/Pacific, Google Meet experienced intermittent issues loading meet.google.com for a total duration of 7 hours 57 minutes.\n- Limited impact in Central Europe occurred between 00:00 - 7:05 US/Pacific.\n- Widespread global impact, with many customers unable to load meet.google.com occurred between 10:25 -11:17 US/Pacific.\nTo our Google Meet customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe initial incident was caused by an overloaded cache service fleet that is part of Google’s Static Content Service (SCS). SCS is used to serve aspects of the Google Meet UI that are not tied to specific to individual users or meetings. If it fails to respond, users will typically be unable to load the Google Meet landing page.\nThe exact reason for the system overload is still being investigated. However, evidence points to a specific set of Meet hot keys bypassed internal caching layers starting on Sep 8, 2025 12:00 AM US/Pacific (9:00 AM Central European Time). This issue affected a limited number of cache servers, leading to their overload, and consequently, an increase in error rates for Google Meet users.\nThe global impact was caused by a configuration change that was attempting to permanently remedy the first issue by configuring a dedicated cache for Google Meet. This dedicated cache was inadequately sized and not pre-warmed. Subsequent traffic spikes resulted in cache misses, which overloaded the cache servers and triggered the global outage.\n## Remediation and Prevention\nGoogle Meet engineers were alerted to the initial outage on 8 September 2025 at 03:12 US/Pacific via an alert indicating increased search activity about Google Meet outages. They immediately began an investigation which identified the overloaded cache service. By 07:05 US/Pacific, engineers had redirected traffic, mitigating the issue for users in Central Europe.\nAt 10:25 US/Pacific, Google engineers rolled out the configuration change to configure a dedicated cache for Google Meet as a permanent fix for the cache overload issue.\nAt 10:38 US/Pacific, Google engineers were alerted to increased error rates for the cache service fleet, and reopened the investigation. Once the global nature of the impact was identified, engineers began rolling back the configuration change at 10:56 US/Pacific.\nThe rollback of the configuration change completed at 11:09 US/Pacific and impact was confirmed to be mitigated by 11:17 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- Implement early detection monitoring and alerting mechanism for proactive future identification.\nTarget: Q3 2025\n- Implement configuration rollout safety improvements to proactively identify load nuances.\nTarget: Q4 2025\n- Validate Static Content Service (SCS) resilience to overload triggers to avoid a recurrence.\nTarget: Q4 2025\n## Detailed Description of Impact\nOn Monday 8 September 2025, Meet users may have experienced issues when attempting to launch or join meetings for a total duration of 7 hours and 57 minutes.\n- Impact Timeframe 1: 00:00 - 07:05 US/Pacific (Impact limited to Central Europe)\n- Impact Timeframe 2: 10:25 - 11:17 US/Pacific (Global Impact)\nWhen a user navigates to meet.google.com and attempts to join a meeting, multiple requests are being made from the user’s browser to SCS. Depending on which request failed, the user will experience different symptoms. Typical symptoms include blank pages and missing controls. Some customers were likely able to successfully reload a page that was broken on the initial load.\nUsers from a total of 1.8 million individual IP addresses were served at least one error when making an SCS request.","status":"AVAILABLE"},{"created":"2025-09-08T20:42:38+00:00","modified":"2025-09-18T17:00:33+00:00","when":"2025-09-08T20:42:38+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 8 September 2025 03:12\n**Incident End:** 8 September 2025 11:18\n**Duration:** 8 hours, 6 minutes\n**Affected Services and Features:** Google Meet\n**Regions/Zones:** Global\n**Description:**\nImpacted users may have experienced intermittent issues loading meet.google.com for a duration of 8 hours 6 minutes.\nThis incident was caused by errors in the system responsible for signing static content, such as Javascript and CSS files. The issue initially impacted users in Germany and possibly in the United Kingdom and Ireland, before spreading globally to all regions.\nThe trigger was identified as a recent code change. Google engineers redirected traffic away from the affected systems to mitigate the European impact. The subsequent global impact was addressed by rolling back the change, which fully mitigated the issue.\nGoogle will complete and share a full Incident Report in the following days that will provide a full root cause analysis.\n**Customer Impact:**\nUsers may have experienced intermittent issues loading meet.google.com.","status":"AVAILABLE"},{"created":"2025-09-08T18:37:45+00:00","modified":"2025-09-08T20:42:38+00:00","when":"2025-09-08T18:37:45+00:00","text":"The issue with Google Meet began at 10:25 PDT on 2025-09-08 and has been resolved for all affected users as of 2025-09-08 11:18 PDT.\nFrom preliminary analysis, the root cause of the issue is a recent change in content edge cache. Our engineers have resolved the issue by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-08T18:30:57+00:00","modified":"2025-09-08T18:37:45+00:00","when":"2025-09-08T18:30:57+00:00","text":"**Summary:**\nResolved issues with Google Meet customers experiencing issues with loading the meet UI\n**Description:**\nOur engineers have identified the root cause and rolled back a recent change.\nCurrent internal telemetry data indicates that the Google Meet service is operating normally as of 11:18 PDT. Our engineers are continuing to monitor the service stability closely.\nWe will provide more information by Monday, 2025-09-08 11:55 PDT.\n**Symptoms:**\nAffected customers experienced slow UI loads and as a result are unable to join meet sessions.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-08T18:07:46+00:00","modified":"2025-09-08T18:30:57+00:00","when":"2025-09-08T18:07:46+00:00","text":"**Summary:**\nSome Google Meet customers are experiencing issues with loading the meet UI\n**Description:**\nWe are experiencing an issue with Google Meet beginning at Monday, 2025-09-08 10:25 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-09-08 11:30 PDT with current details.\n**Symptoms:**\nAffected customers are experiencing slow UI loads and as a result are unable to join meet sessions\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-09-08T17:49:31+00:00","modified":"2025-09-08T18:07:46+00:00","when":"2025-09-08T17:49:31+00:00","text":"**Summary:**\nWe are investigating reports of Google Meet service issues.\n**Description:**\nWe are investigating reports of Google Meet degradation\nWe will provide more information by Monday, 2025-09-08 11:15 PDT.\n**Symptoms:**\nCustomers may be experiencing elevated errors when joining meet sessions.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-09-18T17:00:33+00:00","modified":"2025-09-22T13:48:55+00:00","when":"2025-09-18T17:00:33+00:00","text":"# Incident Report\n## Summary\nOn 8 September 2025, beginning at 00:00 US/Pacific, Google Meet experienced intermittent issues loading meet.google.com for a total duration of 7 hours 57 minutes.\n- Limited impact in Central Europe occurred between 00:00 - 7:05 US/Pacific.\n- Widespread global impact, with many customers unable to load meet.google.com occurred between 10:25 -11:17 US/Pacific.\nTo our Google Meet customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe initial incident was caused by an overloaded cache service fleet that is part of Google’s Static Content Service (SCS). SCS is used to serve aspects of the Google Meet UI that are not tied to specific to individual users or meetings. If it fails to respond, users will typically be unable to load the Google Meet landing page.\nThe exact reason for the system overload is still being investigated. However, evidence points to a specific set of Meet hot keys bypassed internal caching layers starting on Sep 8, 2025 12:00 AM US/Pacific (9:00 AM Central European Time). This issue affected a limited number of cache servers, leading to their overload, and consequently, an increase in error rates for Google Meet users.\nThe global impact was caused by a configuration change that was attempting to permanently remedy the first issue by configuring a dedicated cache for Google Meet. This dedicated cache was inadequately sized and not pre-warmed. Subsequent traffic spikes resulted in cache misses, which overloaded the cache servers and triggered the global outage.\n## Remediation and Prevention\nGoogle Meet engineers were alerted to the initial outage on 8 September 2025 at 03:12 US/Pacific via an alert indicating increased search activity about Google Meet outages. They immediately began an investigation which identified the overloaded cache service. By 07:05 US/Pacific, engineers had redirected traffic, mitigating the issue for users in Central Europe.\nAt 10:25 US/Pacific, Google engineers rolled out the configuration change to configure a dedicated cache for Google Meet as a permanent fix for the cache overload issue.\nAt 10:38 US/Pacific, Google engineers were alerted to increased error rates for the cache service fleet, and reopened the investigation. Once the global nature of the impact was identified, engineers began rolling back the configuration change at 10:56 US/Pacific.\nThe rollback of the configuration change completed at 11:09 US/Pacific and impact was confirmed to be mitigated by 11:17 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- Implement early detection monitoring and alerting mechanism for proactive future identification.\nTarget: Q3 2025\n- Implement configuration rollout safety improvements to proactively identify load nuances.\nTarget: Q4 2025\n- Validate Static Content Service (SCS) resilience to overload triggers to avoid a recurrence.\nTarget: Q4 2025\n## Detailed Description of Impact\nOn Monday 8 September 2025, Meet users may have experienced issues when attempting to launch or join meetings for a total duration of 7 hours and 57 minutes.\n- Impact Timeframe 1: 00:00 - 07:05 US/Pacific (Impact limited to Central Europe)\n- Impact Timeframe 2: 10:25 - 11:17 US/Pacific (Global Impact)\nWhen a user navigates to meet.google.com and attempts to join a meeting, multiple requests are being made from the user’s browser to SCS. Depending on which request failed, the user will experience different symptoms. Typical symptoms include blank pages and missing controls. Some customers were likely able to successfully reload a page that was broken on the initial load.\nUsers from a total of 1.8 million individual IP addresses were served at least one error when making an SCS request.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/muX1XjAhRG1krLmwbQsW"},{"id":"H4H2Z9QdKvqdT92ke1Pb","number":"17539000878079652329","begin":"2025-09-04T17:33:00+00:00","created":"2025-09-04T18:14:04+00:00","end":"2025-09-04T19:45:00+00:00","modified":"2025-09-04T19:59:00+00:00","external_desc":"We're investigating reports of an issue with Google Drive. We will provide more information shortly.","updates":[{"created":"2025-09-04T19:51:10+00:00","modified":"2025-09-04T19:59:00+00:00","when":"2025-09-04T19:51:10+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content\n**Description**\nThe issue with Google Drive has been resolved for all affected users as of Thursday, 2025-09-04 12:45 US/Pacific.\nThe preliminary root cause of the issue was due a recent change rollout and was mitigated by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-09-04T18:21:21+00:00","modified":"2025-09-04T19:51:10+00:00","when":"2025-09-04T18:21:21+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content.\n**Description**\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Thursday, 2025-09-04 13:00 PDT.\nWe will provide more information by Thursday, 2025-09-04 13:00 PDT.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time","status":"SERVICE_DISRUPTION"},{"created":"2025-09-04T18:17:33+00:00","modified":"2025-09-04T18:21:21+00:00","when":"2025-09-04T18:17:33+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content.\n**Description**\nWe are experiencing an issue with Google Drive beginning at Thursday, 2025-09-04 10:33 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 12:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time","status":"SERVICE_DISRUPTION"},{"created":"2025-09-04T18:14:04+00:00","modified":"2025-09-04T18:17:33+00:00","when":"2025-09-04T18:14:04+00:00","text":"We're investigating reports of an issue with Google Drive. We will provide more information shortly.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-04T19:51:10+00:00","modified":"2025-09-04T19:59:00+00:00","when":"2025-09-04T19:51:10+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content\n**Description**\nThe issue with Google Drive has been resolved for all affected users as of Thursday, 2025-09-04 12:45 US/Pacific.\nThe preliminary root cause of the issue was due a recent change rollout and was mitigated by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/H4H2Z9QdKvqdT92ke1Pb"},{"id":"UX3VQZM4R3vLkbSq4Mrf","number":"8467497216017881467","begin":"2025-09-04T07:08:00+00:00","created":"2025-09-04T09:21:05+00:00","end":"2025-09-04T08:12:00+00:00","modified":"2025-09-11T00:29:28+00:00","external_desc":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT. Google Workspace services may not load for impacted users.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","updates":[{"created":"2025-09-11T00:29:28+00:00","modified":"2025-09-11T00:29:28+00:00","when":"2025-09-11T00:29:28+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 4 September, 2025, customer traffic from any source region to end-users served by the Sofia, Bulgaria POP locations may have experienced network disruptions for a total of 12 hours and 5 minutes. A period of severe impact, lasting 1 hour and 4 minutes, resulted in network congestion, elevated latency, and timeouts. Google Workspace services were impacted during this 1 hour and 4 minute period of severe impact.\nWe sincerely apologize for this incident, which does not reflect the level of quality and reliability we strive to offer. We are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nGoogle’s production backbone is a global network that enables connectivity for all user-facing traffic via Points of Presence (POPs) or internet exchanges.\nThe incident was caused by a disruption of multiple fiber paths in the production backbone within Bulgaria which reduced the available network bandwidth for the POP location in Sofia, Bulgaria. Although capacity for POP locations is planned with multiple redundant fiber paths, the remaining capacity for the Sofia, Bulgaria sites to the rest of the backbone could not support the traffic load after the series of fiber failures.\nOur network is designed to be resilient and can automatically handle multiple, isolated fiber cuts by rerouting traffic. In this instance, the network successfully handled the initial fiber failures without significant customer impact. However, a subsequent fiber failure overwhelmed the remaining network capacity, which led to the service disruption.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via our internal monitoring on Thursday, 4 September, 2025 00:19 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear, Google engineers first quickly rerouted serving traffic to adjacent locations, restoring connectivity for most services by approximately Thursday, 4 September, 2025 01:12 US/Pacific.\nSufficient fiber connectivity was restored on Thursday, 4 September, 2025 07:52 US/Pacific and Google engineers began the process of reactivating the capacity, and after validation, allowing traffic serving from the Sofia, Bulgaria POP location.\nAll traffic migrations back to the pre-incident state were completed by Thursday, 4 September, 2025 12:13 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n* To enhance service resilience, we have implemented additional redundancy on the Milan- Sofia fiber path.\n* While adding diversity will improve the overall availability of this site, we are also investigating ways to automatically signal fail-over to our external connection partners when internal capacity reduces beyond a critical threshold to satisfy demand. This will more quickly benefit those customers who connect in multiple regions.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe issue primarily resulted in inaccessibility to Google Workspace services for customers in various locations served by the POP in Sofia, Bulgaria. The impact was first detected at approximately 00:19 US/Pacific on Thursday, 4 September, 2025. Google Workspace users that were routed to the affected POPs would have observed errors and high latency trying to connect to Google services. Additionally, some users encountered DNS resolution issues, including an inability to resolve some publicly accessible Google services.\nGoogle Workspace services were restored by Thursday, 4 September, 2025 01:12 US/Pacific after Google engineers rerouted serving traffic to adjacent locations, restoring connectivity for services. All fiber repairs were completed on Thursday, 4 September, 2025 12:13 US/Pacific.","status":"AVAILABLE"},{"created":"2025-09-05T20:33:50+00:00","modified":"2025-09-11T00:29:28+00:00","when":"2025-09-05T20:33:50+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 4 September 2025 00:08\n**Incident End:** 4 September 2025 01:12\n**Duration:** 1 hour, 4 minutes\n**GWS- Affected Services and Features:**\nPotentially all Google Workspace services may have been impacted for users depending on the point of ingress.\n**Locations:**\nTurkey, Greece, and Bulgaria among other regions\n**Description:**\nGoogle’s production backbone is a global network that enables connectivity for user-facing traffic via Points of Presence (POPs) or internet exchanges.\nA disruption of fiber paths in the production backbone within Bulgaria reduced the available network bandwidth for the POP location in Sofia, Bulgaria. Although capacity for POP locations is planned with multiple redundant fiber paths, the remaining capacity for the Sofia, Bulgaria sites to the rest of the backbone could not support the traffic load after the series of fiber disruptions.\nGoogle engineers mitigated the impact by rerouting serving traffic to adjacent locations, restoring connectivity by approximately Thursday, 4 September, 2025 01:12 US/Pacific.\n**Customer Impact:**\nImpacted users would have encountered network errors and been unable to connect to Google Workspace services.","status":"AVAILABLE"},{"created":"2025-09-04T09:33:40+00:00","modified":"2025-09-05T20:33:50+00:00","when":"2025-09-04T09:33:40+00:00","text":"The issue with Google Workspace services has been resolved for all affected users as of Thursday, 2025-09-04 02:18 PDT.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-09-04T09:21:05+00:00","modified":"2025-09-04T09:33:40+00:00","when":"2025-09-04T09:21:05+00:00","text":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT. Google Workspace services may not load for impacted users.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-11T00:29:28+00:00","modified":"2025-09-11T00:29:28+00:00","when":"2025-09-11T00:29:28+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 4 September, 2025, customer traffic from any source region to end-users served by the Sofia, Bulgaria POP locations may have experienced network disruptions for a total of 12 hours and 5 minutes. A period of severe impact, lasting 1 hour and 4 minutes, resulted in network congestion, elevated latency, and timeouts. Google Workspace services were impacted during this 1 hour and 4 minute period of severe impact.\nWe sincerely apologize for this incident, which does not reflect the level of quality and reliability we strive to offer. We are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nGoogle’s production backbone is a global network that enables connectivity for all user-facing traffic via Points of Presence (POPs) or internet exchanges.\nThe incident was caused by a disruption of multiple fiber paths in the production backbone within Bulgaria which reduced the available network bandwidth for the POP location in Sofia, Bulgaria. Although capacity for POP locations is planned with multiple redundant fiber paths, the remaining capacity for the Sofia, Bulgaria sites to the rest of the backbone could not support the traffic load after the series of fiber failures.\nOur network is designed to be resilient and can automatically handle multiple, isolated fiber cuts by rerouting traffic. In this instance, the network successfully handled the initial fiber failures without significant customer impact. However, a subsequent fiber failure overwhelmed the remaining network capacity, which led to the service disruption.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via our internal monitoring on Thursday, 4 September, 2025 00:19 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear, Google engineers first quickly rerouted serving traffic to adjacent locations, restoring connectivity for most services by approximately Thursday, 4 September, 2025 01:12 US/Pacific.\nSufficient fiber connectivity was restored on Thursday, 4 September, 2025 07:52 US/Pacific and Google engineers began the process of reactivating the capacity, and after validation, allowing traffic serving from the Sofia, Bulgaria POP location.\nAll traffic migrations back to the pre-incident state were completed by Thursday, 4 September, 2025 12:13 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n* To enhance service resilience, we have implemented additional redundancy on the Milan- Sofia fiber path.\n* While adding diversity will improve the overall availability of this site, we are also investigating ways to automatically signal fail-over to our external connection partners when internal capacity reduces beyond a critical threshold to satisfy demand. This will more quickly benefit those customers who connect in multiple regions.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe issue primarily resulted in inaccessibility to Google Workspace services for customers in various locations served by the POP in Sofia, Bulgaria. The impact was first detected at approximately 00:19 US/Pacific on Thursday, 4 September, 2025. Google Workspace users that were routed to the affected POPs would have observed errors and high latency trying to connect to Google services. Additionally, some users encountered DNS resolution issues, including an inability to resolve some publicly accessible Google services.\nGoogle Workspace services were restored by Thursday, 4 September, 2025 01:12 US/Pacific after Google engineers rerouted serving traffic to adjacent locations, restoring connectivity for services. All fiber repairs were completed on Thursday, 4 September, 2025 12:13 US/Pacific.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/UX3VQZM4R3vLkbSq4Mrf"},{"id":"VGfCcPeEwQMdXwDjFYEX","number":"10927389432659257465","begin":"2025-09-04T07:08:00+00:00","created":"2025-09-04T09:16:07+00:00","end":"2025-09-04T08:12:00+00:00","modified":"2025-09-11T00:30:58+00:00","external_desc":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","updates":[{"created":"2025-09-11T00:30:58+00:00","modified":"2025-09-11T00:30:58+00:00","when":"2025-09-11T00:30:58+00:00","text":"A full incident report has been posted at https://google.hw233.vip/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-05T20:58:13+00:00","modified":"2025-09-11T00:30:58+00:00","when":"2025-09-05T20:58:13+00:00","text":"A mini incident report has been posted at https://google.hw233.vip/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-04T09:32:34+00:00","modified":"2025-09-05T20:58:13+00:00","when":"2025-09-04T09:32:34+00:00","text":"The issue with Google Workspace services has been resolved for all affected users as of Thursday, 2025-09-04 02:18 PDT.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-09-04T09:16:07+00:00","modified":"2025-09-04T09:32:34+00:00","when":"2025-09-04T09:16:07+00:00","text":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-11T00:30:58+00:00","modified":"2025-09-11T00:30:58+00:00","when":"2025-09-11T00:30:58+00:00","text":"A full incident report has been posted at https://google.hw233.vip/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"n5hVexuq1PM9onY9qrmo","service_name":"Google Calendar","affected_products":[{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"}],"uri":"incidents/VGfCcPeEwQMdXwDjFYEX"},{"id":"3cc92cDxkMbGrvFaiySR","number":"16411630852503207546","begin":"2025-09-04T07:08:00+00:00","created":"2025-09-04T09:13:39+00:00","end":"2025-09-04T08:12:00+00:00","modified":"2025-09-11T00:31:29+00:00","external_desc":"**Summary**\nMultiple Google Workspace services may be inaccessible to some customers in the EMEA region.\n**Description**\nWe are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace services may not load for impacted users.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-09-11T00:31:29+00:00","modified":"2025-09-11T00:31:29+00:00","when":"2025-09-11T00:31:29+00:00","text":"A full incident report has been posted at https://google.hw233.vip/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-05T20:56:35+00:00","modified":"2025-09-11T00:31:29+00:00","when":"2025-09-05T20:56:35+00:00","text":"A mini incident report has been posted at https://google.hw233.vip/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-04T09:34:34+00:00","modified":"2025-09-05T20:56:35+00:00","when":"2025-09-04T09:34:34+00:00","text":"The issue with Google Workspace services has been resolved for all affected users as of Thursday, 2025-09-04 02:18 PDT.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-09-04T09:13:39+00:00","modified":"2025-09-04T09:34:34+00:00","when":"2025-09-04T09:13:39+00:00","text":"**Summary**\nMultiple Google Workspace services may be inaccessible to some customers in the EMEA region.\n**Description**\nWe are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace services may not load for impacted users.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-11T00:31:29+00:00","modified":"2025-09-11T00:31:29+00:00","when":"2025-09-11T00:31:29+00:00","text":"A full incident report has been posted at https://google.hw233.vip/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/3cc92cDxkMbGrvFaiySR"},{"id":"R6N8442fvbXvxRFsbLd4","number":"11980598393880330639","begin":"2025-08-25T21:27:00+00:00","created":"2025-08-25T22:23:08+00:00","end":"2025-08-25T21:52:00+00:00","modified":"2025-08-25T22:48:49+00:00","external_desc":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors.\n**Description**\nWe were experiencing an issue with Google Docs beginning at Monday, 2025-08-25 14:27 PDT.\nOur engineering team has successfully mitigated the issue and we are currently monitoring our system for stability.\nWe will provide an update by Monday, 2025-08-25 15:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings\n**Workaround**\nNone at this time.","updates":[{"created":"2025-08-25T22:48:12+00:00","modified":"2025-08-25T22:48:49+00:00","when":"2025-08-25T22:48:12+00:00","text":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors\n**Description**\nThe issue with Google Docs has been resolved for all affected users as of Monday, 2025-08-25 14:52 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings.","status":"AVAILABLE"},{"created":"2025-08-25T22:23:08+00:00","modified":"2025-08-25T22:48:12+00:00","when":"2025-08-25T22:23:08+00:00","text":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors.\n**Description**\nWe were experiencing an issue with Google Docs beginning at Monday, 2025-08-25 14:27 PDT.\nOur engineering team has successfully mitigated the issue and we are currently monitoring our system for stability.\nWe will provide an update by Monday, 2025-08-25 15:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-08-25T22:48:12+00:00","modified":"2025-08-25T22:48:49+00:00","when":"2025-08-25T22:48:12+00:00","text":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors\n**Description**\nThe issue with Google Docs has been resolved for all affected users as of Monday, 2025-08-25 14:52 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"wNHuVFtZEWU5Mmj2eRCK","service_name":"Google Docs","affected_products":[{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"}],"uri":"incidents/R6N8442fvbXvxRFsbLd4"},{"id":"jkx7uigB29SrPe8zwvfZ","number":"16032085283414294175","begin":"2025-08-25T16:25:00+00:00","created":"2025-08-25T20:01:27+00:00","end":"2025-08-25T20:25:00+00:00","modified":"2025-08-29T00:55:22+00:00","external_desc":"**Summary:**\nGoogle Voice customers may have been unable to place or receive calls on their Google Voice clients and desk phones\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Monday, 2025-08-25 15:00 PDT.\n**Diagnosis / Customer Symptoms:**\nAffected customers may have experienced dropped calls and connection issues intermittently.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-08-29T00:55:22+00:00","modified":"2025-08-29T00:55:23+00:00","when":"2025-08-29T00:55:22+00:00","text":"# Incident Report\n## Summary\n**Incident Start:** 25 August 2025 09:25 US/Pacific\n**Incident End:** 25 August 2025 13:25 US/Pacific\n**Duration:** 4 hours\nOn Monday, 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nTo our affected customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA software update inadvertently caused a massive number of devices to simultaneously connect to a critical component of our voice infrastructure. This surge of connection attempts overwhelmed the component, resulting in server tasks dropping traffic and repeatedly crashing for approximately 4 hours.\nThe issue was compounded because the devices were configured to immediately and repeatedly retry failed connections, creating a feedback loop that intensified the overload.\nAs a result, this core component was unable to process traffic from other clients, including Google Voice. This resulted in intermittent dropped calls and a significant percentage of call failures during the incident’s peak.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue via an internal monitoring at 07:20 US/Pacific on 25 August and immediately started an investigation. Once the nature and scope of the issue was identified, Google engineers rolled back the software update and increased capacity to the affected servers.These actions successfully mitigated the issue for all affected users by 13:25 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- To prevent service degradation, client applications implement a longer retry backoff upon receiving an HTTP 503 error for connection failures.\n- Improve and update software rollout procedures and tooling to protect against unintentional changes and resulting traffic spikes.\n- Improve capacity planning and server utilization monitoring for the affected servers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nFrom 09:25 to 13:25 US/Pacific on 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nBoth enterprise and consumer users experienced intermittent dropped calls and connection issues across all devices such as mobile, web, and desk phones. At the peak of the incident, call failure rates reached up to 25%, with approximately one call dropped per second due to peer timeouts.","status":"AVAILABLE"},{"created":"2025-08-26T07:46:45+00:00","modified":"2025-08-29T00:55:22+00:00","when":"2025-08-26T07:46:45+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 25 August 2025 09:25\n**Incident End:** 25 August 2025 13:25\n**Duration:** 4 hours\n**Affected Services and Features:**\nGoogle Voice\n**Regions/Zones:** Global\n**Description:**\nGoogle Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours. Based on preliminary analysis, the root cause is a software rollout which resulted in increased traffic and overloaded key infrastructure Google uses for voice calls.\n**Customer Impact:**\nAffected customers intermittently experienced dropped calls and connection issues. This issue may have affected our enterprise and consumer account customers across all devices (mobile, web, deskphones or dedicated hardware).","status":"AVAILABLE"},{"created":"2025-08-25T21:12:05+00:00","modified":"2025-08-26T07:46:45+00:00","when":"2025-08-25T21:12:05+00:00","text":"**Summary:**\nGoogle Voice customers may have been unable to place or receive calls on their Google Voice clients and deskphones\n**Description:**\nThe issue with Google Voice has been resolved for all affected users as of Monday, 2025-08-25 12:57 PDT\nThe issue was due to a hardware device firmware update causing higher than expected traffic to an overloaded server. Our engineers have mitigated the issue by disabling the software change and adding extra capacity to the affected servers.\nWe thank you for your patience while we worked on resolving the issue.\n**Diagnosis / Customer Symptoms:**\nAffected customers may have experienced dropped calls and connection issues intermittently.\n**Workaround:**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-08-25T20:01:27+00:00","modified":"2025-08-26T14:32:50+00:00","when":"2025-08-25T20:01:27+00:00","text":"**Summary:**\nGoogle Voice customers may have been unable to place or receive calls on their Google Voice clients and desk phones\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Monday, 2025-08-25 15:00 PDT.\n**Diagnosis / Customer Symptoms:**\nAffected customers may have experienced dropped calls and connection issues intermittently.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-08-29T00:55:22+00:00","modified":"2025-08-29T00:55:23+00:00","when":"2025-08-29T00:55:22+00:00","text":"# Incident Report\n## Summary\n**Incident Start:** 25 August 2025 09:25 US/Pacific\n**Incident End:** 25 August 2025 13:25 US/Pacific\n**Duration:** 4 hours\nOn Monday, 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nTo our affected customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA software update inadvertently caused a massive number of devices to simultaneously connect to a critical component of our voice infrastructure. This surge of connection attempts overwhelmed the component, resulting in server tasks dropping traffic and repeatedly crashing for approximately 4 hours.\nThe issue was compounded because the devices were configured to immediately and repeatedly retry failed connections, creating a feedback loop that intensified the overload.\nAs a result, this core component was unable to process traffic from other clients, including Google Voice. This resulted in intermittent dropped calls and a significant percentage of call failures during the incident’s peak.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue via an internal monitoring at 07:20 US/Pacific on 25 August and immediately started an investigation. Once the nature and scope of the issue was identified, Google engineers rolled back the software update and increased capacity to the affected servers.These actions successfully mitigated the issue for all affected users by 13:25 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- To prevent service degradation, client applications implement a longer retry backoff upon receiving an HTTP 503 error for connection failures.\n- Improve and update software rollout procedures and tooling to protect against unintentional changes and resulting traffic spikes.\n- Improve capacity planning and server utilization monitoring for the affected servers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nFrom 09:25 to 13:25 US/Pacific on 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nBoth enterprise and consumer users experienced intermittent dropped calls and connection issues across all devices such as mobile, web, and desk phones. At the peak of the incident, call failure rates reached up to 25%, with approximately one call dropped per second due to peer timeouts.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"gh79SrQk6wqonWoxPdCm","service_name":"Google Voice","affected_products":[{"title":"Google Voice","id":"gh79SrQk6wqonWoxPdCm"}],"uri":"incidents/jkx7uigB29SrPe8zwvfZ"},{"id":"B6TDyZ3a2dhAg59QcpZE","number":"18008682940474766646","begin":"2025-08-19T07:00:00+00:00","created":"2025-08-27T19:14:43+00:00","end":"2025-08-28T20:20:00+00:00","modified":"2025-08-28T21:32:44+00:00","external_desc":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nWe are experiencing an issue with ChromeOS M-139.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Wednesday, 2025-08-27 13:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS M-139, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workarounds:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\nOR\n2. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","updates":[{"created":"2025-08-28T21:32:03+00:00","modified":"2025-08-28T21:32:44+00:00","when":"2025-08-28T21:32:03+00:00","text":"Some ChromeOS customers may have experienced issues signing on.\n**Description**\nThe issue with ChromeOS has been resolved for all affected users as of Thursday, 2025-08-28 13:20 PDT.\nFrom preliminary analysis, the issue was triggered by a change in the backend authentication method, which resulted in Single Sign-On (SSO) failures or 2-Step Verification (2SV) to fail for some users.\nAfter identifying the root cause, our engineering team immediately halted the release of the affected Chrome OS/Browser version (16328.55.0/139.0.7258.137) to additional devices. The fix was identified and tested, after which it has been successfully deployed in version 16328.65.0/ 139.0.7258.164.\nCustomers previously affected by this issue may update their Chrome OS/Browser versions to 16328.65.0/139.0.7258.164.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nAuthentication failures were occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue also caused 2-Step Verification (2SV) to fail for some users.","status":"AVAILABLE"},{"created":"2025-08-28T17:21:04+00:00","modified":"2025-08-28T21:32:03+00:00","when":"2025-08-28T17:21:04+00:00","text":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nThis affects only ChromeOS devices on the following OS/Browser Version:\n16328.55.0/139.0.7258.137\nMitigation work is taking longer than anticipated.\nFix is undergoing automated testing. Once completed, engineering will validate the results and we'll make the fix available to users.\nWe currently don't have ETA for the version with the fix to be available.\nWe will provide an update by Thursday, 2025-08-28 17:30 US/Pacific with current details.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workaround:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\nOR\n2. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","status":"SERVICE_INFORMATION"},{"created":"2025-08-27T20:22:22+00:00","modified":"2025-08-28T17:21:04+00:00","when":"2025-08-27T20:22:22+00:00","text":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nThis affects only ChromeOS devices on the following OS/Browser Version:\n16328.55.0/139.0.7258.137\nMitigation work is currently underway by our engineering team.\nAfter identifying the root cause, our engineering team immediately halted the release of the affected version to additional devices. The fix has been identified and tested, and the build for the fix is currently underway.\nThe version with the fix is expected to be available by tomorrow.\nWe will provide an update by Thursday, 2025-08-28 13:30 US/Pacific with current details.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workaround:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\nOR\n2. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","status":"SERVICE_INFORMATION"},{"created":"2025-08-27T19:14:43+00:00","modified":"2025-08-27T20:22:22+00:00","when":"2025-08-27T19:14:43+00:00","text":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nWe are experiencing an issue with ChromeOS M-139.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Wednesday, 2025-08-27 13:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS M-139, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workarounds:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\nOR\n2. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-08-28T21:32:03+00:00","modified":"2025-08-28T21:32:44+00:00","when":"2025-08-28T21:32:03+00:00","text":"Some ChromeOS customers may have experienced issues signing on.\n**Description**\nThe issue with ChromeOS has been resolved for all affected users as of Thursday, 2025-08-28 13:20 PDT.\nFrom preliminary analysis, the issue was triggered by a change in the backend authentication method, which resulted in Single Sign-On (SSO) failures or 2-Step Verification (2SV) to fail for some users.\nAfter identifying the root cause, our engineering team immediately halted the release of the affected Chrome OS/Browser version (16328.55.0/139.0.7258.137) to additional devices. The fix was identified and tested, after which it has been successfully deployed in version 16328.65.0/ 139.0.7258.164.\nCustomers previously affected by this issue may update their Chrome OS/Browser versions to 16328.65.0/139.0.7258.164.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nAuthentication failures were occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue also caused 2-Step Verification (2SV) to fail for some users.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"RjKbsvuGeE7M1pvBHmvX","service_name":"ChromeOS","affected_products":[{"title":"ChromeOS","id":"RjKbsvuGeE7M1pvBHmvX"}],"uri":"incidents/B6TDyZ3a2dhAg59QcpZE"},{"id":"oFcAZTr4EVieF5Fr6Ee9","number":"6540446163670387082","begin":"2025-07-18T15:10:00+00:00","created":"2025-07-18T15:35:43+00:00","end":"2025-07-18T15:56:00+00:00","modified":"2025-07-22T13:52:47+00:00","external_desc":"Various Workspace services are experiencing elevated latency and error rates","updates":[{"created":"2025-07-22T13:52:47+00:00","modified":"2025-07-22T13:52:47+00:00","when":"2025-07-22T13:52:47+00:00","text":"The Incident Report is now available on the https://status.cloud.google.com/incidents/8cY8jdUpEGGbsSMSQk7J\n[Google Cloud Service Health Dashboard] while we prepare a full Issue Summary Report.","status":"AVAILABLE"},{"created":"2025-07-18T22:21:25+00:00","modified":"2025-07-22T13:52:47+00:00","when":"2025-07-18T22:21:25+00:00","text":"The Mini Incident Report is available on the https://status.cloud.google.com/incidents/8cY8jdUpEGGbsSMSQk7J (Google Cloud Service Health Dashboard) while we prepare a full Incident Summary Report.","status":"AVAILABLE"},{"created":"2025-07-18T16:05:39+00:00","modified":"2025-07-18T22:21:25+00:00","when":"2025-07-18T16:05:39+00:00","text":"The issue has been mitigated for all affected users as of 2025-07-18 08:56 US/Pacific.\nA subset of workspace customers experienced elevated latency and error rates using workspace services.\nOur initial investigation points to a hardware infrastructure failure as the likely cause. Engineers have mitigated the workspace impact by routing the traffic away from the affected hardware infrastructure.\nWe thank you for your patience while we're working on resolving the issue.","status":"AVAILABLE"},{"created":"2025-07-18T15:35:43+00:00","modified":"2025-07-18T16:05:39+00:00","when":"2025-07-18T15:35:43+00:00","text":"Various Workspace services are experiencing elevated latency and error rates","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-07-22T13:52:47+00:00","modified":"2025-07-22T13:52:47+00:00","when":"2025-07-22T13:52:47+00:00","text":"The Incident Report is now available on the https://status.cloud.google.com/incidents/8cY8jdUpEGGbsSMSQk7J\n[Google Cloud Service Health Dashboard] while we prepare a full Issue Summary Report.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"},{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/oFcAZTr4EVieF5Fr6Ee9"},{"id":"1hSNYy2ihYPewRrRji6R","number":"14610264225459492371","begin":"2025-07-16T01:27:00+00:00","created":"2025-07-16T18:48:30+00:00","end":"2025-07-17T00:16:00+00:00","modified":"2025-07-18T15:47:54+00:00","external_desc":"**Summary:**\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineering team has identified the source of the issue and is implementing a fix.\nWe will provide more information by Wednesday, 2025-07-16 12:30:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","updates":[{"created":"2025-07-18T08:22:38+00:00","modified":"2025-07-18T15:47:54+00:00","when":"2025-07-18T08:22:38+00:00","text":"# Incident Report\n## Summary\nOn Tuesday, 15 July 2025, for a duration of up to 22 hours and 49 minutes, Google Drive users were denied access to the Google Drive for desktop application. Affected users received an error message ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more’. This message was erroneously triggered and no data was programmatically wiped.\nWe deeply regret the disruption our Google Drive for Desktop users experienced. We recognize the impact this had on your businesses, and we are truly sorry for falling short of your expectations. We have investigated the root cause of the issue and are implementing measures to improve the resilience of our services.\n## Root Cause\nThe root cause of the issue was an update to Drive's policy configuration. As part of this update, the admin setting that restricts Google Drive for desktop to authorized devices had its default unintentionally changed from FALSE to TRUE. This change took effect for domains which hadn't previously overridden the setting.\nAs a result, a number of users were unexpectedly prevented from launching the Google Drive for desktop application, requiring them to restart the application. The impact was limited to Windows and Mac devices used by Enterprise customers. The incident began at 18:27 US/Pacific on 15 July 2025 and deteriorated further on 16 July 2025.\n## Remediation and Prevention\nAt 18:27 US/Pacific on Tuesday, 15 July 2025, a 1% rollout release began causing minor impact. As the release progressed, the situation escalated significantly and became evident on Wednesday, 16 July 2025 from an increase in support cases. Our engineering team was alerted at 09:25 US/Pacific and immediately began the investigation.\nAs the first step to mitigate impact, Google engineers deployed a client-side mitigation to all Google Drive for desktop clients by 11:47 US/Pacific on Wednesday, 16 July 2025. This deployment disabled Enterprise Mobility Management support within Google Drive for Desktop. While this aimed to prevent further application launch blocks, it wasn't guaranteed to reach all devices before the block signal.\nConcurrently, a policy fix was deployed via a rollback. This rollback targeted the incorrect policy configuration that triggered the blocks, which completed at 17:16 US/Pacific on Wednesday, 16 July 2025.\nWe are deeply sorry for the length and severity of this incident. Google is committed to preventing a repeat of this incident in the future and we are completing the following actions :\n* Improve the error message displayed to our users when devices that are not company owned are blocked from using Google Drive for desktop.\n* Improve monitoring on the number and rate of policy related actions taken by Google Drive for desktop.\n## Detailed Description of Impact\nOn Tuesday, 15 July 2025 from 18:27 US/Pacific to Wednesday, 16 July 2025, 17:16 US/Pacific:\n#### Google Drive\nApproximately 1% users may have been prevented from launching Google Drive for desktop and encountered the error ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.’\nAs a temporary workaround, administrators were advised to manually disable the ‘Only allow Google Drive for desktop on authorized devices’ setting to prevent blocks on their managed devices.\n## Additional Information for Customers\nOnce the incident was mitigated, users who exited the application without taking any further action were able to access the application after they relaunched it using existing account configuration. Some users who may have disconnected their account using the button that was presented as an option, were able to regain access after signing into the application and performing additional actions to reconfigure any mirror roots and re-pin any files that required offline availability. These users who disconnected their account may also have had to download the files again that were previously cached.","status":"AVAILABLE"},{"created":"2025-07-17T17:41:09+00:00","modified":"2025-07-18T08:22:38+00:00","when":"2025-07-17T17:41:09+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 15 July 2025 18:27\n**Incident End:** 16 July 2025 18:19\n**Duration:** 23 hours, 52 minutes\n**Affected Services and Features:**\nGoogle Drive for desktop\n**Regions/Zones:** Global\n**Description:**\nA subset of Google Drive users who had their account in Drive for desktop disabled, observed the error: “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more”. Despite the message, no Google Drive content was actually wiped from the users’ computers. The message was a result of a configuration change to the default admin policies. Google will complete a full Incident Report in the following days.\n**Customer Impact:**\nA subset of Google Drive users had their account in Drive for desktop disabled and observed the error: “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more”. Despite the message, no Google Drive content was actually wiped from the users’ computers.","status":"AVAILABLE"},{"created":"2025-07-17T02:05:13+00:00","modified":"2025-07-17T17:41:09+00:00","when":"2025-07-17T02:05:13+00:00","text":"The issue with Google Drive for Desktop has been resolved for all affected users as of Wednesday, 2025-07-16 17:16 PDT.\nThe issue was due to a configuration change to the default admin policies. Our engineers mitigated the issue by rolling back the configuration.\n**Action required:** If any users are still experiencing issues, they should quit the Drive for desktop app and relaunch it.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-07-17T01:09:00+00:00","modified":"2025-07-17T02:05:13+00:00","when":"2025-07-17T01:09:00+00:00","text":"**Summary**:\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description**:\nThe engineering team believes that the incident mitigation is complete and is confirming its effectiveness.\nAn estimated time of resolution provided is Wednesday, 2025-07-16 at 19:00:00 PDT.\nWe will provide more information by Wednesday, 2025-07-16 19:30:00 PDT with additional details.\n**Diagnosis / Customer Symptoms**:\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround**:\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE_DISRUPTION"},{"created":"2025-07-16T19:50:49+00:00","modified":"2025-07-17T01:09:00+00:00","when":"2025-07-16T19:50:49+00:00","text":"**Summary:**\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for Desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineers have implemented a mitigation strategy and have partially mitigated the issue.\nThe engineering team continues to work towards full mitigation.\nAn estimated time of resolution provided is Wednesday, 2025-07-16 at 17:00:00 PDT.\nWe will provide more information by Wednesday, 2025-07-16 18:00:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE_DISRUPTION"},{"created":"2025-07-16T19:25:42+00:00","modified":"2025-07-16T19:50:49+00:00","when":"2025-07-16T19:25:42+00:00","text":"**Summary:**\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineering team has identified the source of the issue and is working on implementing a fix.\nWe will provide more information by Wednesday, 2025-07-16 13:30:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE_DISRUPTION"},{"created":"2025-07-16T18:48:30+00:00","modified":"2025-07-16T19:27:53+00:00","when":"2025-07-16T18:48:30+00:00","text":"**Summary:**\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineering team has identified the source of the issue and is implementing a fix.\nWe will provide more information by Wednesday, 2025-07-16 12:30:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-07-18T08:22:38+00:00","modified":"2025-07-18T15:47:54+00:00","when":"2025-07-18T08:22:38+00:00","text":"# Incident Report\n## Summary\nOn Tuesday, 15 July 2025, for a duration of up to 22 hours and 49 minutes, Google Drive users were denied access to the Google Drive for desktop application. Affected users received an error message ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more’. This message was erroneously triggered and no data was programmatically wiped.\nWe deeply regret the disruption our Google Drive for Desktop users experienced. We recognize the impact this had on your businesses, and we are truly sorry for falling short of your expectations. We have investigated the root cause of the issue and are implementing measures to improve the resilience of our services.\n## Root Cause\nThe root cause of the issue was an update to Drive's policy configuration. As part of this update, the admin setting that restricts Google Drive for desktop to authorized devices had its default unintentionally changed from FALSE to TRUE. This change took effect for domains which hadn't previously overridden the setting.\nAs a result, a number of users were unexpectedly prevented from launching the Google Drive for desktop application, requiring them to restart the application. The impact was limited to Windows and Mac devices used by Enterprise customers. The incident began at 18:27 US/Pacific on 15 July 2025 and deteriorated further on 16 July 2025.\n## Remediation and Prevention\nAt 18:27 US/Pacific on Tuesday, 15 July 2025, a 1% rollout release began causing minor impact. As the release progressed, the situation escalated significantly and became evident on Wednesday, 16 July 2025 from an increase in support cases. Our engineering team was alerted at 09:25 US/Pacific and immediately began the investigation.\nAs the first step to mitigate impact, Google engineers deployed a client-side mitigation to all Google Drive for desktop clients by 11:47 US/Pacific on Wednesday, 16 July 2025. This deployment disabled Enterprise Mobility Management support within Google Drive for Desktop. While this aimed to prevent further application launch blocks, it wasn't guaranteed to reach all devices before the block signal.\nConcurrently, a policy fix was deployed via a rollback. This rollback targeted the incorrect policy configuration that triggered the blocks, which completed at 17:16 US/Pacific on Wednesday, 16 July 2025.\nWe are deeply sorry for the length and severity of this incident. Google is committed to preventing a repeat of this incident in the future and we are completing the following actions :\n* Improve the error message displayed to our users when devices that are not company owned are blocked from using Google Drive for desktop.\n* Improve monitoring on the number and rate of policy related actions taken by Google Drive for desktop.\n## Detailed Description of Impact\nOn Tuesday, 15 July 2025 from 18:27 US/Pacific to Wednesday, 16 July 2025, 17:16 US/Pacific:\n#### Google Drive\nApproximately 1% users may have been prevented from launching Google Drive for desktop and encountered the error ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.’\nAs a temporary workaround, administrators were advised to manually disable the ‘Only allow Google Drive for desktop on authorized devices’ setting to prevent blocks on their managed devices.\n## Additional Information for Customers\nOnce the incident was mitigated, users who exited the application without taking any further action were able to access the application after they relaunched it using existing account configuration. Some users who may have disconnected their account using the button that was presented as an option, were able to regain access after signing into the application and performing additional actions to reconfigure any mirror roots and re-pin any files that required offline availability. These users who disconnected their account may also have had to download the files again that were previously cached.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/1hSNYy2ihYPewRrRji6R"},{"id":"owtYrdrh7EwXtfAmyu3U","number":"3633811992985588295","begin":"2025-07-02T18:40:00+00:00","created":"2025-07-02T20:17:49+00:00","end":"2025-07-02T20:02:00+00:00","modified":"2025-07-02T21:10:25+00:00","external_desc":"We have received reports of an intermittent issue with Google Meet beginning at Wednesday, 2025-07-02 11:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-07-02 13:45 PDT with current details.\nWe apologize to all who are affected by the disruption.","updates":[{"created":"2025-07-02T21:10:22+00:00","modified":"2025-07-02T21:10:25+00:00","when":"2025-07-02T21:10:22+00:00","text":"A small number of users may have experienced intermittent issues connecting to or creating new Google Meet sessions.","status":"AVAILABLE"},{"created":"2025-07-02T20:49:20+00:00","modified":"2025-07-02T21:10:22+00:00","when":"2025-07-02T20:49:20+00:00","text":"Our engineering team has applied the mitigation and our internal monitoring indicates recovery. Engineering team continues to closely monitor the environment.\nWe will provide an update by Wednesday, 2025-07-02 14:45 PDT with current details.","status":"SERVICE_INFORMATION"},{"created":"2025-07-02T20:17:49+00:00","modified":"2025-07-02T20:49:20+00:00","when":"2025-07-02T20:17:49+00:00","text":"We have received reports of an intermittent issue with Google Meet beginning at Wednesday, 2025-07-02 11:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-07-02 13:45 PDT with current details.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-07-02T21:10:22+00:00","modified":"2025-07-02T21:10:25+00:00","when":"2025-07-02T21:10:22+00:00","text":"A small number of users may have experienced intermittent issues connecting to or creating new Google Meet sessions.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/owtYrdrh7EwXtfAmyu3U"},{"id":"Eab7zGLWSaUJyh9c9R9L","number":"1558320242640410399","begin":"2025-06-12T17:58:00+00:00","created":"2025-06-12T19:01:47+00:00","end":"2025-06-12T23:00:00+00:00","modified":"2025-06-14T00:07:48+00:00","external_desc":"We're investigating reports of an issue with Gmail, Google Calendar, Google Chat, Google Cloud Search, Google Docs, Google Drive, Google Meet, Google Tasks, and Google Voice. We will provide more information shortly.","updates":[{"created":"2025-06-13T23:59:37+00:00","modified":"2025-06-14T00:07:48+00:00","when":"2025-06-13T23:59:37+00:00","text":"The detailed Incident Report is now available on the Google Cloud Service Health Dashboard at https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW","status":"AVAILABLE"},{"created":"2025-06-13T07:00:35+00:00","modified":"2025-06-13T23:59:37+00:00","when":"2025-06-13T07:00:35+00:00","text":"# Mini Incident Report\nWe are deeply sorry for the impact to all of our users and their customers that this service disruption/outage caused. Businesses large and small trust Google Cloud with your workloads and we will do better. In the coming days, we will publish a full incident report of the root cause, detailed timeline and robust remediation steps we will be taking. Given the size and impact of this incident, we would like to provide some information below.\nPlease note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n**(All Times US/Pacific)**\n**Incident Start:** 12 June, 2025 10:49\n**All regions except us-central1 mitigated:** 12 June, 2025 12:48\n**Incident End:** 12 June, 2025 13:49\n**Duration:** 3 hours\n**Regions/Zones:** Global\n**Description:**\nMultiple Google Cloud and Google Workspace products experienced increased 503 errors in external API requests, impacting customers.\nFrom our initial analysis, the issue occurred due to an invalid automated quota update to our API management system which was distributed globally, causing external API requests to be rejected. To recover we bypassed the offending quota check, which allowed recovery in most regions within 2 hours. However, the quota policy database in us-central1 became overloaded, resulting in much longer recovery in that region. Several products had moderate residual impact (e.g. backlogs) for up to an hour after the primary issue was mitigated and a small number recovering after that.\nGoogle will complete a full Incident Report in the following days that will provide a detailed root cause.\n**Customer Impact:**\nCustomers had intermittent API and user-interface access issues to the impacted services. Existing streaming and IaaS resources were not impacted.\n**Additional details:**\nThis incident should not have happened, and we will take the following measures to prevent future recurrence:\nPrevent our API management platform from failing due to invalid or corrupt data.\nPrevent metadata from propagating globally without appropriate protection, testing and monitoring in place.\nImprove system error handling and comprehensive testing for handling of invalid data.\n**Affected Services and Features:**\n**Google Cloud Products:**\n* Identity and Access Management\n* Cloud Build\n* Cloud Key Management Service\n* Google Cloud Storage\n* Cloud Monitoring\n* Google Cloud Dataproc\n* Cloud Security Command Center\n* Artifact Registry\n* Cloud Workflows\n* Cloud Healthcare\n* Resource Manager API\n* Dataproc Metastore\n* Cloud Run\n* VMWare engine\n* Dataplex\n* Migrate to Virtual Machines\n* Google BigQuery\n* Contact Center AI Platform\n* Google Cloud Deploy\n* Media CDN\n* Colab Enterprise\n* Vertex Gemini API\n* Cloud Data Fusion\n* Cloud Asset Inventory\n* Datastream\n* Integration Connectors\n* Apigee\n* Google Cloud NetApp Volumes\n* Google Cloud Bigtable\n* Looker (Google Cloud core)\n* Looker Studio\n* Google Cloud Functions\n* Cloud Load Balancing\n* Traffic Director\n* Document AI\n* AutoML Translation\n* Pub/Sub Lite\n* API Gateway\n* Agent Assist\n* AlloyDB for PostgreSQL\n* Cloud Firestore\n* Cloud Logging\n* Cloud Shell\n* Cloud Memorystore\n* Cloud Spanner\n* Contact Center Insights\n* Database Migration Service\n* Dialogflow CX\n* Dialogflow ES\n* Google App Engine\n* Google Cloud Composer\n* Google Cloud Console\n* Google Cloud DNS\n* Google Cloud Pub/Sub\n* Google Cloud SQL\n* Google Compute Engine\n* Identity Platform\n* Managed Service for Apache Kafka\n* Memorystore for Memcached\n* Memorystore for Redis\n* Memorystore for Redis Cluster\n* Persistent Disk\n* Personalized Service Health\n* Speech-to-Text\n* Text-to-Speech\n* Vertex AI Search\n* Retail API\n* Vertex AI Feature Store\n* BigQuery Data Transfer Service\n* Google Cloud Marketplace\n* Cloud NAT\n* Hybrid Connectivity\n* Cloud Vision\n* Network Connectivity Center\n* Cloud Workstations\n**Google Workspace Products:**\n* AppSheet\n* Gmail\n* Google Calendar\n* Google Drive\n* Google Chat\n* Google Voice\n* Google Docs\n* Google Meet\n* Google Cloud Search\n* Google Tasks","status":"AVAILABLE"},{"created":"2025-06-12T19:53:15+00:00","modified":"2025-06-13T07:00:35+00:00","when":"2025-06-12T19:53:15+00:00","text":"The issue with Google Chat, Google Meet, Gmail, Google Calendar, Google Drive, Google Cloud Search, Google Tasks, Google Voice has been resolved for all affected users as of Thursday, 2025-06-12 12:48 PDT.\nThe issue with AppSheet has been resolved for all affected users as of Thursday, 2025-06-12 16:00 PDT.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-06-12T19:30:51+00:00","modified":"2025-06-12T19:53:15+00:00","when":"2025-06-12T19:30:51+00:00","text":"All product impacts except Google Meet have recovered.\nGoogle engineers continue to work on full mitigation.","status":"SERVICE_DISRUPTION"},{"created":"2025-06-12T19:01:47+00:00","modified":"2025-06-12T19:30:51+00:00","when":"2025-06-12T19:01:47+00:00","text":"We're investigating reports of an issue with Gmail, Google Calendar, Google Chat, Google Cloud Search, Google Docs, Google Drive, Google Meet, Google Tasks, and Google Voice. We will provide more information shortly.\nMultiple Workspace products beginning on Thursday, at 2025-06-12 10:58 PDT may be experiencing service issues.\nOur engineers are currently investigating the issue.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-06-13T23:59:37+00:00","modified":"2025-06-14T00:07:48+00:00","when":"2025-06-13T23:59:37+00:00","text":"The detailed Incident Report is now available on the Google Cloud Service Health Dashboard at https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"},{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Cloud Search","id":"iJPnujhwzow3TaPE1dM8"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"},{"title":"Google Tasks","id":"Vups7kniH52V5x4UX1gM"},{"title":"Google Voice","id":"gh79SrQk6wqonWoxPdCm"}],"uri":"incidents/Eab7zGLWSaUJyh9c9R9L"},{"id":"KjFLjecwg9xeewnUYjh1","number":"15423978574401052094","begin":"2025-06-09T22:07:00+00:00","created":"2025-06-09T23:28:15+00:00","end":"2025-06-10T00:00:00+00:00","modified":"2025-06-12T12:39:18+00:00","external_desc":"**Summary:**\nGoogle Classroom experiencing elevated latency issue\n**Description:**\nWe are experiencing an Classroom beginning at Monday, 2025-06-09 15:00 PDT\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-06-09 17:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nThe impacted users would experience increased latency while using Classroom.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-06-12T12:39:18+00:00","modified":"2025-06-12T12:39:18+00:00","when":"2025-06-12T12:39:18+00:00","text":"# Incident Report\n## Summary\nOn Monday, 9 June 2025, at 15:07 US/Pacific, Google Classroom experienced increased latency for a duration of 1 hour 53 minutes. We deeply regret the disruption our Google Classroom users experienced. We know you rely on our services, and we fell short of meeting your expectations. We are taking immediate action to improve our platform's reliability and prevent future disruptions.\n## Root Cause\nThe incident was triggered by a schema change applied to Google's globally distributed database, which stores and serves a majority of Classroom data. This schema change caused a surge in CPU utilization across multiple clusters, reaching the maximum utilization at 100 percent. As a result, a backend service relying on the database experienced a significant drop in availability and performance, impacting Google Classroom users.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via a monitoring alert on June 9 at 15:07 US/Pacific and immediately started an investigation. The issue was mitigated on 9 June at 17:00 US/Pacific once the schema change was fully rolled back.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\nUpdate Google Classrooms database release process efficiency to use incremental rollout, to aid in identifying potential impact and prevent a recurrence.\nImplement proactive identification for schema changes with potential wide-reaching impact prior to rollout.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\n### Google Classroom\nOn 9 June 2025, from 15:07 to 17:00 US/Pacific Google Classroom users experienced increased latency wherein the application may have failed to load for an extended period.\nGoogle Classroom experienced a significant drop in availability and performance for approximately ~0.1% of users for a duration of 1 hour and 53 minutes. Most requests made to the database experienced latency or unavailability during the incident time.","status":"AVAILABLE"},{"created":"2025-06-10T04:02:24+00:00","modified":"2025-06-12T12:39:18+00:00","when":"2025-06-10T04:02:24+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 9 June, 2025 15:07\n**Incident End:** 9 June, 2025 17:00\n**Duration:** 1 hour, 53 minutes\n**Affected Services and Features:**\nGoogle Classroom\n**Regions/Zones:** Global\n**Description:**\nGoogle Classroom users experienced elevated latency for a duration of 1 hour and 53 minutes. The incident was triggered by a schema change applied to Google's globally distributed database. This schema change caused a surge in utilization across multiple zones, reaching the maximum utilization at 100%. As a result, a backend service relying on the database experienced a significant drop in availability and performance, impacting Classroom users. The incident was mitigated by rolling back the schema change.\nGoogle will complete a detailed Incident report in the following days that will provide a full root cause.\n**Customer Impact:**\nImpacted users experienced increased latency while using Google Classroom.","status":"AVAILABLE"},{"created":"2025-06-10T00:12:23+00:00","modified":"2025-06-10T04:02:24+00:00","when":"2025-06-10T00:12:23+00:00","text":"**Summary:**\nGoogle Classroom experienced elevated latency issue\n**Description:**\nThe issue with Classroom has been resolved for all affected users as of Monday, 2025-06-09 17:00 PDT.\nThe issue was due to a recent change in the Spanner Database which prevented many users from being able to access Classroom. Our engineers have mitigated the issue by rolling back the recent release to the previous version.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Diagnosis:**\nThe impacted users experienced increased latency while using Classroom.\n**Workaround:**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-06-09T23:51:12+00:00","modified":"2025-06-10T00:12:23+00:00","when":"2025-06-09T23:51:12+00:00","text":"**Summary:**\nGoogle Classroom experiencing elevated latency issue\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Monday, 2025-06-09 17:50 PDT.\n**Diagnosis:**\nThe impacted users would experience increased latency while using Classroom.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-06-09T23:28:15+00:00","modified":"2025-06-09T23:51:12+00:00","when":"2025-06-09T23:28:15+00:00","text":"**Summary:**\nGoogle Classroom experiencing elevated latency issue\n**Description:**\nWe are experiencing an Classroom beginning at Monday, 2025-06-09 15:00 PDT\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-06-09 17:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nThe impacted users would experience increased latency while using Classroom.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-06-12T12:39:18+00:00","modified":"2025-06-12T12:39:18+00:00","when":"2025-06-12T12:39:18+00:00","text":"# Incident Report\n## Summary\nOn Monday, 9 June 2025, at 15:07 US/Pacific, Google Classroom experienced increased latency for a duration of 1 hour 53 minutes. We deeply regret the disruption our Google Classroom users experienced. We know you rely on our services, and we fell short of meeting your expectations. We are taking immediate action to improve our platform's reliability and prevent future disruptions.\n## Root Cause\nThe incident was triggered by a schema change applied to Google's globally distributed database, which stores and serves a majority of Classroom data. This schema change caused a surge in CPU utilization across multiple clusters, reaching the maximum utilization at 100 percent. As a result, a backend service relying on the database experienced a significant drop in availability and performance, impacting Google Classroom users.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via a monitoring alert on June 9 at 15:07 US/Pacific and immediately started an investigation. The issue was mitigated on 9 June at 17:00 US/Pacific once the schema change was fully rolled back.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\nUpdate Google Classrooms database release process efficiency to use incremental rollout, to aid in identifying potential impact and prevent a recurrence.\nImplement proactive identification for schema changes with potential wide-reaching impact prior to rollout.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\n### Google Classroom\nOn 9 June 2025, from 15:07 to 17:00 US/Pacific Google Classroom users experienced increased latency wherein the application may have failed to load for an extended period.\nGoogle Classroom experienced a significant drop in availability and performance for approximately ~0.1% of users for a duration of 1 hour and 53 minutes. Most requests made to the database experienced latency or unavailability during the incident time.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"3GoNVrVz48jeNZSyLJMV","service_name":"Classroom","affected_products":[{"title":"Classroom","id":"3GoNVrVz48jeNZSyLJMV"}],"uri":"incidents/KjFLjecwg9xeewnUYjh1"},{"id":"gwMkZKeZCE1eAC6z5EvH","number":"7701776215533702872","begin":"2025-05-22T17:00:00+00:00","created":"2025-05-23T02:02:34+00:00","end":"2025-05-23T01:45:00+00:00","modified":"2025-05-23T04:41:12+00:00","external_desc":"**Summary**\nAppSheet application and Dashboards are not showing within Looker Studio.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-05-22 10:00 US/Pacific.\nOur engineering team identified the cause was due to a recent change and are actively working on the rollback.\nThe mitigation is expected to complete by Thursday, 2025-05-22 19:45 US/Pacific.\nWe will provide more information by Thursday, 2025-05-22 20:15 US/Pacific.\n**Customer Symptoms**\nAppSheet apps will not show within Looker Studio and Dashboards will return Error.\n**Workaround**\nNone at this moment","updates":[{"created":"2025-05-23T04:40:46+00:00","modified":"2025-05-23T04:41:12+00:00","when":"2025-05-23T04:40:46+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 22 May, 2025 10:00\n**Incident End:** 22 May, 2025 18:45\n**Duration:** 8 hours, 45 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:** Global\n**Description:**\nAppSheet in Looker Studio experienced issues where applications and dashboards were not visible. Based on our analysis, this was caused by a recent change that AppSheet introduced to enhance its authentication for the Looker Studio connector. Once the impact was identified by our Engineering team, we promptly rolled back the change, restoring the connector's stability.\n**Customer Impact:**\nImpacted customers may have experienced AppSheet apps not showing within Looker Studio, and dependent dashboards may have returned an exception: ‘Community Connector Error’.","status":"AVAILABLE"},{"created":"2025-05-23T02:15:46+00:00","modified":"2025-05-23T04:40:46+00:00","when":"2025-05-23T02:15:46+00:00","text":"**Description**\nThe issue with AppSheet has been resolved for all affected users as of Thursday, 2025-05-22 19:05 US/Pacific.\nA recent change introduced by our engineering team caused the issue. To mitigate the impact, the change was rolled back, and the stability of the service has been confirmed.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-05-23T02:02:34+00:00","modified":"2025-05-23T02:15:46+00:00","when":"2025-05-23T02:02:34+00:00","text":"**Summary**\nAppSheet application and Dashboards are not showing within Looker Studio.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-05-22 10:00 US/Pacific.\nOur engineering team identified the cause was due to a recent change and are actively working on the rollback.\nThe mitigation is expected to complete by Thursday, 2025-05-22 19:45 US/Pacific.\nWe will provide more information by Thursday, 2025-05-22 20:15 US/Pacific.\n**Customer Symptoms**\nAppSheet apps will not show within Looker Studio and Dashboards will return Error.\n**Workaround**\nNone at this moment","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-05-23T04:40:46+00:00","modified":"2025-05-23T04:41:12+00:00","when":"2025-05-23T04:40:46+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 22 May, 2025 10:00\n**Incident End:** 22 May, 2025 18:45\n**Duration:** 8 hours, 45 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:** Global\n**Description:**\nAppSheet in Looker Studio experienced issues where applications and dashboards were not visible. Based on our analysis, this was caused by a recent change that AppSheet introduced to enhance its authentication for the Looker Studio connector. Once the impact was identified by our Engineering team, we promptly rolled back the change, restoring the connector's stability.\n**Customer Impact:**\nImpacted customers may have experienced AppSheet apps not showing within Looker Studio, and dependent dashboards may have returned an exception: ‘Community Connector Error’.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/gwMkZKeZCE1eAC6z5EvH"},{"id":"SNDHnqEixWu9UKSJNjrR","number":"11844245915375502324","begin":"2025-05-13T15:08:00+00:00","created":"2025-05-13T15:47:25+00:00","end":"2025-05-13T18:36:00+00:00","modified":"2025-05-13T19:06:00+00:00","external_desc":"**Summary:**\nMultiple Workspace services are experiencing elevated error rates\n**Description:**\nBeginning at 08:08 US/Pacific time on Tuesday, May 13, 2025, there is an ongoing issue affecting Google Docs, Google Sheets, Google Slides, Gmail, Google Chat, and Google Meet.\nInternal monitoring indicates that Google Docs was recovered as of 08:15 US/Pacific time, and the engineering team is still investigating the issue for the remaining impact.\nWe will provide an update by Tuesday, 2025-05-13 09:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services\n**Workaround:**\nCustomers can retry the errored operation.","updates":[{"created":"2025-05-13T19:05:47+00:00","modified":"2025-05-13T19:06:00+00:00","when":"2025-05-13T19:05:47+00:00","text":"Google Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nGoogle Chat issues were resolved as of 11:36 US/Pacific time.\nPreliminary analysis indicates a datacenter power issue overloaded the backend storage system, causing intermittent customer traffic disruptions. Automated systems recovered some services, while engineers mitigated the remaining impact by rerouting traffic from the affected datacenter and also provisioned additional resources.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-05-13T18:21:37+00:00","modified":"2025-05-13T19:05:47+00:00","when":"2025-05-13T18:21:37+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nGoogle Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nGoogle Chat mitigation efforts are ongoing. Engineers are implementing various actions and observing a gradual decrease in impact.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Tuesday, 2025-05-13 12:45 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T17:39:35+00:00","modified":"2025-05-13T18:21:37+00:00","when":"2025-05-13T17:39:35+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nGoogle Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nMitigation work is currently underway by our engineering team for Google Chat.\nThe mitigation is expected to complete by Tuesday, 2025-05-13 11:15 US/Pacific.\nWe will provide an update by Tuesday, 2025-05-13 11:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T16:22:00+00:00","modified":"2025-05-13T17:39:35+00:00","when":"2025-05-13T16:22:00+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nInternal monitoring indicates that various workspace services are now recovered.\nGoogle Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nMitigation work is currently underway by our engineering team for Google Chat.\nThe mitigation is expected to complete by Tuesday, 2025-05-13 09:45 US/Pacific.\nOur engineers are continuing to investigate Gmail SMTP related issues.\nWe will provide an update by Tuesday, 2025-05-13 10:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T15:58:11+00:00","modified":"2025-05-13T16:22:00+00:00","when":"2025-05-13T15:58:11+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nInternal monitoring indicates that Google Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time, and the engineering team is still investigating the issue for the remaining impact for Google Chat, Gmail, and Google Meet.\nWe will provide an update by Tuesday, 2025-05-13 10:00 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T15:47:25+00:00","modified":"2025-05-13T15:58:11+00:00","when":"2025-05-13T15:47:25+00:00","text":"**Summary:**\nMultiple Workspace services are experiencing elevated error rates\n**Description:**\nBeginning at 08:08 US/Pacific time on Tuesday, May 13, 2025, there is an ongoing issue affecting Google Docs, Google Sheets, Google Slides, Gmail, Google Chat, and Google Meet.\nInternal monitoring indicates that Google Docs was recovered as of 08:15 US/Pacific time, and the engineering team is still investigating the issue for the remaining impact.\nWe will provide an update by Tuesday, 2025-05-13 09:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-05-13T19:05:47+00:00","modified":"2025-05-13T19:06:00+00:00","when":"2025-05-13T19:05:47+00:00","text":"Google Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nGoogle Chat issues were resolved as of 11:36 US/Pacific time.\nPreliminary analysis indicates a datacenter power issue overloaded the backend storage system, causing intermittent customer traffic disruptions. Automated systems recovered some services, while engineers mitigated the remaining impact by rerouting traffic from the affected datacenter and also provisioned additional resources.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"},{"title":"Google Sheets","id":"GsNjeYHsogeRgTiC5coU"},{"title":"Google Slides","id":"DvWBgkXVhodA3WYwgZoB"}],"uri":"incidents/SNDHnqEixWu9UKSJNjrR"},{"id":"q2AU1kpKWNdBCDtAyjLi","number":"15997718865756572109","begin":"2025-05-13T00:45:00+00:00","created":"2025-05-13T03:42:11+00:00","end":"2025-05-13T04:13:00+00:00","modified":"2025-05-13T06:46:36+00:00","external_desc":"**Summary**\nGoogle Cloud Workspace customers are experiencing an error while using https://gemini.google.com\n**Description**\nWe are experiencing an issue with Gemini beginning at Monday, 2025-05-12 20:04 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-05-12 22:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWhen accessing https://gemini.google.com, the users may observe an error, \"Gemini is currently on a break. Give it a moment and try again later.\"\n**Workaround**\nNone at this time.","updates":[{"created":"2025-05-13T06:38:47+00:00","modified":"2025-05-13T06:39:10+00:00","when":"2025-05-13T06:38:47+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 12 May, 2025 17:45\n**Incident End:** 12 May, 2025 21:13\n**Duration:** 3 hour 28 minutes\n**Affected Services and Features:**\nGemini\n**Description:**\nGoogle Cloud Workspace customers experienced an error while using https://gemini.google.com. From preliminary analysis, the root cause of the issue was a recent code change and Google engineering quickly mitigated the issue by rolling back the code change.\n**Customer Impact:**\nWhen accessing https://gemini.google.com, the users observed an error, \"Gemini is currently on a break. Give it a moment and try again later.\"","status":"AVAILABLE"},{"created":"2025-05-13T04:27:12+00:00","modified":"2025-05-13T06:46:20+00:00","when":"2025-05-13T04:27:12+00:00","text":"**Summary**\nGoogle Cloud Workspace customers are experiencing an error while using https://gemini.google.com\n**Closing Message**\nThe issue with Gemini has been resolved for all affected users as of Monday, 2025-05-12 21:13 US/Pacific.\nGoogle engineers identified the recent code change causing this issue and mitigated the issue by rolling back the changes.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_OUTAGE"},{"created":"2025-05-13T03:42:11+00:00","modified":"2025-05-13T06:46:36+00:00","when":"2025-05-13T03:42:11+00:00","text":"**Summary**\nGoogle Cloud Workspace customers are experiencing an error while using https://gemini.google.com\n**Description**\nWe are experiencing an issue with Gemini beginning at Monday, 2025-05-12 20:04 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-05-12 22:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWhen accessing https://gemini.google.com, the users may observe an error, \"Gemini is currently on a break. Give it a moment and try again later.\"\n**Workaround**\nNone at this time.","status":"SERVICE_OUTAGE"}],"most_recent_update":{"created":"2025-05-13T06:38:47+00:00","modified":"2025-05-13T06:39:10+00:00","when":"2025-05-13T06:38:47+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 12 May, 2025 17:45\n**Incident End:** 12 May, 2025 21:13\n**Duration:** 3 hour 28 minutes\n**Affected Services and Features:**\nGemini\n**Description:**\nGoogle Cloud Workspace customers experienced an error while using https://gemini.google.com. From preliminary analysis, the root cause of the issue was a recent code change and Google engineering quickly mitigated the issue by rolling back the code change.\n**Customer Impact:**\nWhen accessing https://gemini.google.com, the users observed an error, \"Gemini is currently on a break. Give it a moment and try again later.\"","status":"AVAILABLE"},"status_impact":"SERVICE_OUTAGE","severity":"high","service_key":"npdyhgECDJ6tB66MxXyo","service_name":"Gemini","affected_products":[{"title":"Gemini","id":"npdyhgECDJ6tB66MxXyo"}],"uri":"incidents/q2AU1kpKWNdBCDtAyjLi"},{"id":"SwA8chvNSrGA9WPLtyYu","number":"4385767702209736005","begin":"2025-05-07T14:47:00+00:00","created":"2025-05-07T15:33:33+00:00","end":"2025-05-07T22:12:00+00:00","modified":"2025-05-14T00:30:35+00:00","external_desc":"**Summary:**\nGoogle Drive and Gmail customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-05-07 09:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nSome PDF files in Google Drive and Gmail are not being displayed properly\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-05-12T21:46:07+00:00","modified":"2025-05-14T00:30:35+00:00","when":"2025-05-12T21:46:07+00:00","text":"# Incident Report\n## Summary\nOn Friday, 2 May 2025, Gmail, Google Chat, and Google Drive experienced an issue where some customers using Android devices and web browsers were unable to properly display Portable Document Format (PDF) files globally. Generative AI features associated with the impacted PDF files were also affected, for a duration of 5 days, 12 hours, and 4 minutes.\nTo our customers whose workflows were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform's performance and availability.\n## Root Cause\nThe root cause of the issue was an update to an underlying software library that affected how PDF files were interpreted during the rendering process.\nA recent software update to an underlying, third-party library inadvertently introduced a change in how PDF files were interpreted during the rendering process. This change led to some Gmail, Google Chat, and Google Drive users on Android devices and web browsers seeing incomplete or inaccurate content within certain PDFs. This issue also impacted the generative AI features for the affected PDF files. iOS users were unaffected.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage on Wednesday, 7 May at 05:52 US/Pacific via a customer support case and immediately started an investigation.\nOnce the root cause was identified, the issue was mitigated by reverting to the previous software version, which restored the correct rendering of PDFs. The incident was fully mitigated on Wednesday, 7 May 16:21 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n- Add comprehensive test coverage for all new features and critical existing functionality.\n- Improve release-time validation of code changes.\n## Detailed Description of Impact\n- Google Drive: PDF files in drive were not displayed correctly due to rendering issues. Furthermore, Gemini was also affected by the same rendering issue as the Drive viewer for some PDFs.\n- Gmail: PDF files in email were not displayed correctly due to rendering issues.\n- Google Chat: PDF files in chat were not displayed correctly due to rendering issues.","status":"AVAILABLE"},{"created":"2025-05-08T11:19:13+00:00","modified":"2025-05-12T21:46:07+00:00","when":"2025-05-08T11:19:13+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\n**Incident Start:** 30 Apr 2025, 09:00\n**Incident End:** 7 May 2025, 15:12\n**Duration:** 7 days, 6 hours, 12 minutes\n**Affected Services and Features:**\nGoogle Drive, Gmail, Google Chat\n**Regions/Zones:** Global\n**Description:**\nSome PDF files in Google Drive, Gmail and Google Chat were not displayed properly for customers using Android devices and web browsers. The issue also affected the generative AI features for the affected PDF files. This occurred for a duration of 7 days, 6 hours, 12 minutes. The root cause of the incident was a code change in an internal service. This update triggered the incorrect rendering of information in PDFs, impacting users of services dependent on this rendering service. The impact was mitigated by rolling back the internal services to a previous version.\nGoogle will complete a full Incident Report in the following days that will provide a full root cause analysis.\n**Customer Impact:**\nSome PDF files in Google Drive, Gmail, and Google Chat were not displayed properly for customers using android devices and web browsers.\nThe issue also affected the generative AI features for the affected PDF files.\nThe issue did not affect customers using iOS devices.","status":"AVAILABLE"},{"created":"2025-05-08T08:52:01+00:00","modified":"2025-05-08T11:19:13+00:00","when":"2025-05-08T08:52:01+00:00","text":"The problem with Gmail, Google Chat, and Google Drive has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"SERVICE_INFORMATION"},{"created":"2025-05-08T01:05:17+00:00","modified":"2025-05-08T08:52:01+00:00","when":"2025-05-08T01:05:17+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nThe recent issue with PDF rendering affecting Gmail, Google Chat, Google Drive (Web and Android) have been fully resolved.\nGemini responses for some pdfs which were uploaded to Google Drive between April 23rd and May 7th would have degraded responses in the Gemini side panel on Gmail, Google Chat and Google Drive web apps.\nOur engineering team continues to work on mitigating the Gen AI functionality and is expected to complete by Thursday, 2025-05-08 18:00 US/Pacific.\nWe will provide an update by Thursday, 2025-05-08 19:00 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues. If Gemini response is degraded users can use a copy of original pdf and that would work.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T22:34:17+00:00","modified":"2025-05-08T01:05:17+00:00","when":"2025-05-07T22:34:17+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nWe believe the issue with Gmail Web, Google Chat Web, Google Drive Web is fully resolved.\nOur engineers are working on mitigating the issue for Gmail, Google Chat and Google Drive Android customers.\nFull resolution is expected to complete complete by Wednesday, 2025-05-07 18:30 US/Pacific.\nWe will provide an update by Wednesday, 2025-05-07 19:00 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T20:01:50+00:00","modified":"2025-05-07T22:34:17+00:00","when":"2025-05-07T20:01:50+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nWe believe the issue with Gmail Web , Google Chat Web is fully resolved.\nOur engineers are working on mitigating the issue for Gmail and Google Chat Android customers, and Google Drive Android and Web customers.\nFull resolution is expected to complete complete by Wednesday, 2025-05-07 15:30 US/Pacific.\nWe will provide an update by Wednesday, 2025-05-07 15:45 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T17:58:24+00:00","modified":"2025-05-07T20:01:50+00:00","when":"2025-05-07T17:58:24+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nWe believe the issue with Gmail , Google Chat Web is fully resolved.\nOur engineers are working on mitigating the issue for Gmail and Google Chat Android customers, and Google Drive Android and Web customers.\nFull resolution is expected to complete by Wednesday, 2025-05-07 12:30 US/Pacific.\nWe will provide an update by Wednesday, 2025-05-07 12:45 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T16:13:25+00:00","modified":"2025-05-07T17:58:24+00:00","when":"2025-05-07T16:13:25+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail, and Google Chat beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Wednesday, 2025-05-07 11:00 US/Pacific.\nWe will provide more information by Wednesday, 2025-05-07 11:15 US/Pacific.\n**Customer Symptoms:**\nSome PDF files in Google Drive , Gmail, and Google Chat are not being displayed properly\n**Workaround:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T15:48:14+00:00","modified":"2025-05-07T16:13:25+00:00","when":"2025-05-07T15:48:14+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail, and Google Chat beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Wednesday, 2025-05-07 11:00 US/Pacific.\nWe will provide more information by Wednesday, 2025-05-07 11:15 US/Pacific.\n**Customer Symptoms:**\nSome PDF files in Google Drive , Gmail, and Google Chat are not being displayed properly\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T15:33:33+00:00","modified":"2025-05-07T15:48:14+00:00","when":"2025-05-07T15:33:33+00:00","text":"**Summary:**\nGoogle Drive and Gmail customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-05-07 09:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nSome PDF files in Google Drive and Gmail are not being displayed properly\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-05-12T21:46:07+00:00","modified":"2025-05-14T00:30:35+00:00","when":"2025-05-12T21:46:07+00:00","text":"# Incident Report\n## Summary\nOn Friday, 2 May 2025, Gmail, Google Chat, and Google Drive experienced an issue where some customers using Android devices and web browsers were unable to properly display Portable Document Format (PDF) files globally. Generative AI features associated with the impacted PDF files were also affected, for a duration of 5 days, 12 hours, and 4 minutes.\nTo our customers whose workflows were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform's performance and availability.\n## Root Cause\nThe root cause of the issue was an update to an underlying software library that affected how PDF files were interpreted during the rendering process.\nA recent software update to an underlying, third-party library inadvertently introduced a change in how PDF files were interpreted during the rendering process. This change led to some Gmail, Google Chat, and Google Drive users on Android devices and web browsers seeing incomplete or inaccurate content within certain PDFs. This issue also impacted the generative AI features for the affected PDF files. iOS users were unaffected.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage on Wednesday, 7 May at 05:52 US/Pacific via a customer support case and immediately started an investigation.\nOnce the root cause was identified, the issue was mitigated by reverting to the previous software version, which restored the correct rendering of PDFs. The incident was fully mitigated on Wednesday, 7 May 16:21 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n- Add comprehensive test coverage for all new features and critical existing functionality.\n- Improve release-time validation of code changes.\n## Detailed Description of Impact\n- Google Drive: PDF files in drive were not displayed correctly due to rendering issues. Furthermore, Gemini was also affected by the same rendering issue as the Drive viewer for some PDFs.\n- Gmail: PDF files in email were not displayed correctly due to rendering issues.\n- Google Chat: PDF files in chat were not displayed correctly due to rendering issues.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/SwA8chvNSrGA9WPLtyYu"},{"id":"fV4x9UvX32Uv4oWEzKU8","number":"2601336434679622824","begin":"2025-04-16T12:33:00+00:00","created":"2025-04-16T13:22:54+00:00","end":"2025-04-16T13:16:00+00:00","modified":"2025-04-16T13:56:49+00:00","external_desc":"We're investigating reports of an issue with Google Workspace Support. We will provide more information shortly. The affected users are able to access Google Workspace Support, but are seeing error messages, high latency, and/or other unexpected behavior.","updates":[{"created":"2025-04-16T13:56:49+00:00","modified":"2025-04-16T13:56:50+00:00","when":"2025-04-16T13:56:49+00:00","text":"The problem with Google Workspace Support has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-04-16T13:22:54+00:00","modified":"2025-04-16T13:56:49+00:00","when":"2025-04-16T13:22:54+00:00","text":"We're investigating reports of an issue with Google Workspace Support. We will provide more information shortly. The affected users are able to access Google Workspace Support, but are seeing error messages, high latency, and/or other unexpected behavior. Customer symptoms: Customers may experience an issue with creating cases. Workaround: Customers may attempt to create a case again.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-04-16T13:56:49+00:00","modified":"2025-04-16T13:56:50+00:00","when":"2025-04-16T13:56:49+00:00","text":"The problem with Google Workspace Support has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/fV4x9UvX32Uv4oWEzKU8"},{"id":"zPuCXFfb6pM1yftqcozU","number":"2645351609659327365","begin":"2025-04-06T04:04:00+00:00","created":"2025-04-06T07:09:05+00:00","end":"2025-04-06T10:30:00+00:00","modified":"2025-04-06T11:34:04+00:00","external_desc":"**Summary:**\nChat support functionality is degraded for Google Workspace Support customers.\n**Description:**\nWe are experiencing an issue with Google Workspace Support beginning on Saturday, 2025-04-05 21:04 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Sunday, 2025-04-06 01:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nCustomers will be unable to contact Google Workspace support via Chat support.\n**Workaround:** Impacted customers can create a support case for any assistance.","updates":[{"created":"2025-04-06T11:34:04+00:00","modified":"2025-04-06T11:34:04+00:00","when":"2025-04-06T11:34:04+00:00","text":"The problem with Google Workspace Support has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-04-06T10:40:24+00:00","modified":"2025-04-06T11:34:04+00:00","when":"2025-04-06T10:40:24+00:00","text":"**Summary:** Chat support functionality is currently unavailable for Google Workspace Support customers.\n**Description:**\nOur engineering team is currently validating a fix that has been applied by the vendor partner.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Sunday, 2025-04-06 05:00 US/Pacific.\n**Diagnosis:** Customers will be unable to contact Google Workspace support via Chat support.\n**Workaround:** Impacted customers can create a support case or utilize email support functionality for any assistance.","status":"SERVICE_DISRUPTION"},{"created":"2025-04-06T08:47:16+00:00","modified":"2025-04-06T10:40:24+00:00","when":"2025-04-06T08:47:16+00:00","text":"**Summary:** Chat support functionality is currently unavailable for Google Workspace Support customers.\n**Description:**\nOur engineering team still continues to investigate the issue with vendor partner.\nWe will provide an update by Sunday, 2025-04-06 04:00 US/Pacific with current details.\n**Diagnosis**: Customers will be unable to contact Google Workspace support via Chat support.\n**Workaround:** Impacted customers can create a support case or utilize email support functionality for any assistance.","status":"SERVICE_DISRUPTION"},{"created":"2025-04-06T07:55:42+00:00","modified":"2025-04-06T08:47:16+00:00","when":"2025-04-06T07:55:42+00:00","text":"**Summary:** Chat support functionality is currently unavailable for Google Workspace Support customers.\n**Description:**\nOur engineering team continues to investigate the issue with vendor partner.\nWe will provide an update by Sunday, 2025-04-06 02:00 US/Pacific with current details.\n**Diagnosis:** Customers will be unable to contact Google Workspace support via Chat support.\n**Workaround:** Impacted customers can create a support case for any assistance.","status":"SERVICE_DISRUPTION"},{"created":"2025-04-06T07:09:05+00:00","modified":"2025-04-06T07:55:42+00:00","when":"2025-04-06T07:09:05+00:00","text":"**Summary:**\nChat support functionality is degraded for Google Workspace Support customers.\n**Description:**\nWe are experiencing an issue with Google Workspace Support beginning on Saturday, 2025-04-05 21:04 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Sunday, 2025-04-06 01:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nCustomers will be unable to contact Google Workspace support via Chat support.\n**Workaround:** Impacted customers can create a support case for any assistance.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-04-06T11:34:04+00:00","modified":"2025-04-06T11:34:04+00:00","when":"2025-04-06T11:34:04+00:00","text":"The problem with Google Workspace Support has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/zPuCXFfb6pM1yftqcozU"},{"id":"bmtU6Df9dGVrZXXBHW1F","number":"735815451923338399","begin":"2025-03-25T18:00:00+00:00","created":"2025-03-25T18:00:02+00:00","end":"2025-04-03T16:21:00+00:00","modified":"2025-04-03T17:31:07+00:00","external_desc":"Calendar users in Chile's Aysén region need to update the time zone for all events taking place after 2025-04-05 which are impacted by the new DST rules.\nOn 2025-03-20, Chile's Aysén region adopted a new time zone (IANA: \"America/Coyhaique”). This time zone will diverge from the continental Chile time zone “Chile Time” (IANA: “America/Santiago”) after 2025-04-05, by staying at UTC-03.\nUsers in, or operating in, the Aysén region will be impacted: their events taking place after 2025-04-05 will be, by default, in the Chile time zone and will diverge by one hour compared to the new Aysén region local time.\nImpacted users are recommended to take 2 actions as soon as possible:\n* Go to Calendar Settings \u003e General \u003e Time zone and set their Primary time zone to “Punta Arenas time” (IANA: \"America/Punta_Arenas”).\n* Update all their events taking place after 2025-04-05 in this region to the “Punta Arenas time”.\nAdditionally, we are working on introducing a new time zone for the Aysén region.\nWe will post a communication update when it will be available in Calendar.\nWe will provide an update by Friday, 2025-04-04 13:00 US/Pacific with current details.\nIf impacted users don’t take action, after 2025-04-05 their calendar will:\n* Show the wrong time with the “Now” red line on Calendar Web;\n* Display all events shifted by one hour;\n* Create new events in the wrong time zone.","updates":[{"created":"2025-04-03T16:35:27+00:00","modified":"2025-04-03T17:31:07+00:00","when":"2025-04-03T16:35:27+00:00","text":"**Summary**\nCalendar users in Chile's Aysén region need to update the time zone for all events taking place after 2025-04-05 which are impacted by the new DST rules.\n**Description**\nThe new timezone, \"Coyhaique”, introduced for Chile's Aysén region is now available in Calendar.\nOn 2025-03-20, Chile's Aysén region adopted a new time zone (IANA: \"America/Coyhaique”). This time zone will diverge from the continental Chile time zone “Chile Time” (IANA: “America/Santiago”) after 2025-04-05, by staying at UTC-03.\nUsers in, or operating in, the Aysén region that have not already switched to \"Punta Arenas time” will be impacted: their events taking place after 2025-04-05 will be, by default, in the Chile time zone and will diverge by one hour compared to the new Aysén region local time.\nImpacted users are recommended to take 2 actions as soon as possible:\n**1)** Go to Calendar Settings \u003e General \u003e Time zone and set their Primary time zone to “Coyhaique” (IANA: \"America/Coyhaique”).\n**2)** Update all their events taking place after 2025-04-05 in this region to the “Coyhaique” time zone.\nUsers that have updated their settings to \"Punta Arenas time” should update to the new \"Coyhaique” to ensure that potential future changes to the local time are correctly applied.\nThe name \"Coyhaique” for the new time zone is provisional and may change in the future.\nCalendar users on Android and iOS devices should use “Punta Arenas time” until their operating system is updated to have the new time zone.\n**Diagnosis**\nIf impacted users don’t take action and are still using “Chile time”, after 2025-04-05 their calendar will:\n- Show the wrong time with the “Now” red line on Calendar Web;\n- Display all events shifted by one hour;\n- Create new events in the wrong time zone.\nUsers that updated to \"Punta Arenas time” should not see any disruption, but not updating to \"Coyhaique” would expose them to disruption should the two time zones diverge in the future.\n**Workaround**\nImpacted users that still use \"Chile time” should take the 2 actions described above. Impacted users that applied the previously suggested fix should update to \"Coyhaique” when convenient.","status":"AVAILABLE"},{"created":"2025-03-25T18:00:02+00:00","modified":"2025-04-03T16:35:27+00:00","when":"2025-03-25T18:00:02+00:00","text":"Calendar users in Chile's Aysén region need to update the time zone for all events taking place after 2025-04-05 which are impacted by the new DST rules.\nOn 2025-03-20, Chile's Aysén region adopted a new time zone (IANA: \"America/Coyhaique”). This time zone will diverge from the continental Chile time zone “Chile Time” (IANA: “America/Santiago”) after 2025-04-05, by staying at UTC-03.\nUsers in, or operating in, the Aysén region will be impacted: their events taking place after 2025-04-05 will be, by default, in the Chile time zone and will diverge by one hour compared to the new Aysén region local time.\nImpacted users are recommended to take 2 actions as soon as possible:\n* Go to Calendar Settings \u003e General \u003e Time zone and set their Primary time zone to “Punta Arenas time” (IANA: \"America/Punta_Arenas”).\n* Update all their events taking place after 2025-04-05 in this region to the “Punta Arenas time”.\nAdditionally, we are working on introducing a new time zone for the Aysén region.\nWe will post a communication update when it will be available in Calendar.\nWe will provide an update by Friday, 2025-04-04 13:00 US/Pacific with current details.\nIf impacted users don’t take action, after 2025-04-05 their calendar will:\n* Show the wrong time with the “Now” red line on Calendar Web;\n* Display all events shifted by one hour;\n* Create new events in the wrong time zone.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-04-03T16:35:27+00:00","modified":"2025-04-03T17:31:07+00:00","when":"2025-04-03T16:35:27+00:00","text":"**Summary**\nCalendar users in Chile's Aysén region need to update the time zone for all events taking place after 2025-04-05 which are impacted by the new DST rules.\n**Description**\nThe new timezone, \"Coyhaique”, introduced for Chile's Aysén region is now available in Calendar.\nOn 2025-03-20, Chile's Aysén region adopted a new time zone (IANA: \"America/Coyhaique”). This time zone will diverge from the continental Chile time zone “Chile Time” (IANA: “America/Santiago”) after 2025-04-05, by staying at UTC-03.\nUsers in, or operating in, the Aysén region that have not already switched to \"Punta Arenas time” will be impacted: their events taking place after 2025-04-05 will be, by default, in the Chile time zone and will diverge by one hour compared to the new Aysén region local time.\nImpacted users are recommended to take 2 actions as soon as possible:\n**1)** Go to Calendar Settings \u003e General \u003e Time zone and set their Primary time zone to “Coyhaique” (IANA: \"America/Coyhaique”).\n**2)** Update all their events taking place after 2025-04-05 in this region to the “Coyhaique” time zone.\nUsers that have updated their settings to \"Punta Arenas time” should update to the new \"Coyhaique” to ensure that potential future changes to the local time are correctly applied.\nThe name \"Coyhaique” for the new time zone is provisional and may change in the future.\nCalendar users on Android and iOS devices should use “Punta Arenas time” until their operating system is updated to have the new time zone.\n**Diagnosis**\nIf impacted users don’t take action and are still using “Chile time”, after 2025-04-05 their calendar will:\n- Show the wrong time with the “Now” red line on Calendar Web;\n- Display all events shifted by one hour;\n- Create new events in the wrong time zone.\nUsers that updated to \"Punta Arenas time” should not see any disruption, but not updating to \"Coyhaique” would expose them to disruption should the two time zones diverge in the future.\n**Workaround**\nImpacted users that still use \"Chile time” should take the 2 actions described above. Impacted users that applied the previously suggested fix should update to \"Coyhaique” when convenient.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"n5hVexuq1PM9onY9qrmo","service_name":"Google Calendar","affected_products":[{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"}],"uri":"incidents/bmtU6Df9dGVrZXXBHW1F"},{"id":"LJWrQTaxDnPkvZmyT1xW","number":"11114999909125435187","begin":"2025-03-18T23:55:00+00:00","created":"2025-03-19T02:23:20+00:00","end":"2025-03-19T03:32:00+00:00","modified":"2025-03-19T03:56:25+00:00","external_desc":"**Summary:**\nVideo Playbacks are failing for Google Drive Web.\n**Description**\nWe believe the issue with Google Drive is partially resolved.\nFull resolution is expected to complete by Tuesday, 2025-03-18 19:30 US/Pacific.\nWe will provide an update by Tuesday, 2025-03-18 21:00 US/Pacific with current details.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive Web users would encounter an infinite spinner during playbacks of the videos. Mobile application users are not impacted.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-03-19T03:55:31+00:00","modified":"2025-03-19T03:56:25+00:00","when":"2025-03-19T03:55:31+00:00","text":"The issue with Google Drive has been resolved for all affected users as of Tuesday, 2025-03-18 20:32 US/Pacific.\nA recent change rolled out in the video player infrastructure inadvertently prevented the player from fully loading. Our engineering team performed a rollback of this change successfully to restore the player's loading functionality.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-03-19T03:23:48+00:00","modified":"2025-03-19T03:55:31+00:00","when":"2025-03-19T03:23:48+00:00","text":"**Summary:**\nVideo Playbacks are failing for Google Drive Web.\n**Description:**\nWe believe the issue with Google Drive is resolved for the majority of customers.\nOur engineering team continues the mitigation efforts to achieve full resolution for all impacted customers. We do not have an ETA for the complete mitigation at this time.\nWe will provide an update by Tuesday, 2025-03-18 23:00 US/Pacific with current details.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive Web users would encounter an infinite spinner during playbacks of the videos. Mobile application users are not impacted.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-03-19T02:23:20+00:00","modified":"2025-03-19T03:23:48+00:00","when":"2025-03-19T02:23:20+00:00","text":"**Summary:**\nVideo Playbacks are failing for Google Drive Web.\n**Description**\nWe believe the issue with Google Drive is partially resolved.\nFull resolution is expected to complete by Tuesday, 2025-03-18 19:30 US/Pacific.\nWe will provide an update by Tuesday, 2025-03-18 21:00 US/Pacific with current details.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive Web users would encounter an infinite spinner during playbacks of the videos. Mobile application users are not impacted.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-03-19T03:55:31+00:00","modified":"2025-03-19T03:56:25+00:00","when":"2025-03-19T03:55:31+00:00","text":"The issue with Google Drive has been resolved for all affected users as of Tuesday, 2025-03-18 20:32 US/Pacific.\nA recent change rolled out in the video player infrastructure inadvertently prevented the player from fully loading. Our engineering team performed a rollback of this change successfully to restore the player's loading functionality.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/LJWrQTaxDnPkvZmyT1xW"},{"id":"88AgyzSZoEV9wrHwsEoe","number":"13715942946582871514","begin":"2025-03-11T17:58:00+00:00","created":"2025-03-11T19:49:45+00:00","end":"2025-03-11T19:13:00+00:00","modified":"2025-03-11T20:26:43+00:00","external_desc":"Google Workspace customers can create chats during case creation but experience an issue providing attachments for chat cases","updates":[{"created":"2025-03-11T20:25:08+00:00","modified":"2025-03-11T20:26:43+00:00","when":"2025-03-11T20:25:08+00:00","text":"The issue with Google Workspace Support is believed to be affecting a very small number of customers between 10:28 and 12:13 US/Pacific. At this time no customers are impacted that we know of but our Engineering team continues to investigate the issue. If you have questions or are impacted, please open a case with the Support Team and we will work with you until this issue is resolved. No further updates will be provided here. We thank you for your patience while we're working on resolving the issue.","status":"AVAILABLE"},{"created":"2025-03-11T19:49:45+00:00","modified":"2025-03-11T20:25:08+00:00","when":"2025-03-11T19:49:45+00:00","text":"Google Workspace customers can create chats during case creation but experience an issue providing attachments for chat cases","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-03-11T20:25:08+00:00","modified":"2025-03-11T20:26:43+00:00","when":"2025-03-11T20:25:08+00:00","text":"The issue with Google Workspace Support is believed to be affecting a very small number of customers between 10:28 and 12:13 US/Pacific. At this time no customers are impacted that we know of but our Engineering team continues to investigate the issue. If you have questions or are impacted, please open a case with the Support Team and we will work with you until this issue is resolved. No further updates will be provided here. We thank you for your patience while we're working on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/88AgyzSZoEV9wrHwsEoe"},{"id":"pJzo6KcR37eV8bCLdXgS","number":"15392475158423179555","begin":"2025-03-10T10:50:00+00:00","created":"2025-03-10T13:12:14+00:00","end":"2025-03-10T13:30:00+00:00","modified":"2025-03-13T12:15:06+00:00","external_desc":"We're investigating reports of an issue with NotebookLM. We will provide more information shortly. The affected users are unable to access NotebookLM.","updates":[{"created":"2025-03-13T12:15:06+00:00","modified":"2025-03-13T12:15:06+00:00","when":"2025-03-13T12:15:06+00:00","text":"# Incident Report\n## Summary\nOn Monday, March 10, 2025, at 03:50 US/Pacific, NotebookLM experienced an outage where users were unable to use chat, upload sources, or use interactive audio features globally for 2 hours and 40 minutes.\nWe deeply regret the disruption our Google Workspace customers experienced. We know you rely on our services, and we fell short of meeting your expectations. We are committed to improving our platform's reliability and performance, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe root cause has been determined to be permission errors that impeded NotebookLM's access to the necessary underlying model.\nSubsequent to a recent configuration change, the model was assigned to incorrect access control lists (ACLs) by the internal router that manages the models. This change resulted in the NotebookLM model being associated with a default model family possessing more stringent access controls.\nTherefore, the NotebookLM model encountered permission errors, and incoming requests were rejected, culminating in customer inability to utilize the impacted features.\n## Remediation and Prevention\nGoogle engineers were alerted of the outage from our internal monitoring system on March 10, 2025, at 03:50 US/Pacific, and immediately started an investigation. Once the underlying cause was identified, Google engineers added the required access controls to the default model at 06:30 US/Pacific to fully mitigate the impact.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n* Implement a failsafe mechanism on the internal system that manages the Machine Learning (ML) models. This will prevent similar issues from occurring due to unintended configuration changes.\n* Updating the configuration of the router to bypass the access control check when models are moved to the internal system.\n## Detailed Description of Impact\nBetween 03:50 US/Pacific and 06:30 US/Pacific on Monday, 10 March 2025, NotebookLM and Enterprise users were unable to use chat, upload sources, or use interactive audio. Customers experienced errors up to 100% due to this incident related to permission errors.","status":"AVAILABLE"},{"created":"2025-03-10T18:09:22+00:00","modified":"2025-03-13T12:15:06+00:00","when":"2025-03-10T18:09:22+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\nIncident Start: 10 March 2025 03:50\nIncident End: 10 March 2025 06:30\nDuration: 2 hours, 40 minutes\nAffected Services and Features:\n* Notebook LM\nRegions/Zones: Global\nDescription:\nNotebookLM users were unable to use chat, upload sources, or use interactive audio globally for a duration of 2 hours and 40 minutes.\nThe preliminary root cause of the issue was due to a recent change where a service was missing permissions to interact with another service. As a result of these missing permissions, we rejected incoming requests, leading to users being unable to use the affected features.\nEngineers identified the root cause within the permissions system, and implemented a code change to resolve the issue.\nWe apologize for any inconvenience this may have caused. Google will complete a full Incident Report in the following days that will provide a full root cause analysis.\nCustomer Impact:\n* NotebookLM users were unable to use chat, upload sources, or interactive audio.","status":"AVAILABLE"},{"created":"2025-03-10T14:19:48+00:00","modified":"2025-03-10T18:09:22+00:00","when":"2025-03-10T14:19:48+00:00","text":"The problem with NotebookLM has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-03-10T13:12:14+00:00","modified":"2025-03-10T14:19:48+00:00","when":"2025-03-10T13:12:14+00:00","text":"We're investigating reports of an issue with NotebookLM. We will provide more information shortly. The affected users are unable to access NotebookLM. NotebookLM users are unable to use chat, upload sources or use interactive audio None at this time","status":"SERVICE_OUTAGE"}],"most_recent_update":{"created":"2025-03-13T12:15:06+00:00","modified":"2025-03-13T12:15:06+00:00","when":"2025-03-13T12:15:06+00:00","text":"# Incident Report\n## Summary\nOn Monday, March 10, 2025, at 03:50 US/Pacific, NotebookLM experienced an outage where users were unable to use chat, upload sources, or use interactive audio features globally for 2 hours and 40 minutes.\nWe deeply regret the disruption our Google Workspace customers experienced. We know you rely on our services, and we fell short of meeting your expectations. We are committed to improving our platform's reliability and performance, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe root cause has been determined to be permission errors that impeded NotebookLM's access to the necessary underlying model.\nSubsequent to a recent configuration change, the model was assigned to incorrect access control lists (ACLs) by the internal router that manages the models. This change resulted in the NotebookLM model being associated with a default model family possessing more stringent access controls.\nTherefore, the NotebookLM model encountered permission errors, and incoming requests were rejected, culminating in customer inability to utilize the impacted features.\n## Remediation and Prevention\nGoogle engineers were alerted of the outage from our internal monitoring system on March 10, 2025, at 03:50 US/Pacific, and immediately started an investigation. Once the underlying cause was identified, Google engineers added the required access controls to the default model at 06:30 US/Pacific to fully mitigate the impact.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n* Implement a failsafe mechanism on the internal system that manages the Machine Learning (ML) models. This will prevent similar issues from occurring due to unintended configuration changes.\n* Updating the configuration of the router to bypass the access control check when models are moved to the internal system.\n## Detailed Description of Impact\nBetween 03:50 US/Pacific and 06:30 US/Pacific on Monday, 10 March 2025, NotebookLM and Enterprise users were unable to use chat, upload sources, or use interactive audio. Customers experienced errors up to 100% due to this incident related to permission errors.","status":"AVAILABLE"},"status_impact":"SERVICE_OUTAGE","severity":"high","service_key":"sqTm5ZmzCmb66kvyzcNS","service_name":"NotebookLM","affected_products":[{"title":"NotebookLM","id":"sqTm5ZmzCmb66kvyzcNS"}],"uri":"incidents/pJzo6KcR37eV8bCLdXgS"},{"id":"H8N8DzxWuLAMcZ2Dgjdj","number":"13875903582452877296","begin":"2025-02-17T15:06:00+00:00","created":"2025-02-17T23:17:56+00:00","end":"2025-02-17T20:51:00+00:00","modified":"2025-02-17T23:25:55+00:00","external_desc":"### **Summary:**\nSome customers were unable to send SMS on Google Voice.\n### **Description:**\nWe have experienced an issue with Google Voice beginning on Monday, 2025-02-17 at 07:06 US/Pacific.\nOur engineering team has applied the fix which has successfully mitigated the issue.\nWe apologize to all who were affected by the disruption.\n### **Diagnosis:**\nImpacted customers may have received an error message stating ‘The message violates our Acceptable Use Policy’.\nThe issue has impacted only consumer users and not Google Workspace users.\n### **Workaround:**\nCustomers may retry to send the SMS which might have resolved the issue. Sending SMS messages from Android or iOS devices appeared to be functioning correctly.","updates":[{"created":"2025-02-17T23:25:55+00:00","modified":"2025-02-17T23:25:56+00:00","when":"2025-02-17T23:25:55+00:00","text":"The issue with Google Voice has been resolved for all affected users as of Monday, 2025-02-17 12:51 US/Pacific.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-02-17T23:17:56+00:00","modified":"2025-02-17T23:25:55+00:00","when":"2025-02-17T23:17:56+00:00","text":"### **Summary:**\nSome customers were unable to send SMS on Google Voice.\n### **Description:**\nWe have experienced an issue with Google Voice beginning on Monday, 2025-02-17 at 07:06 US/Pacific.\nOur engineering team has applied the fix which has successfully mitigated the issue.\nWe apologize to all who were affected by the disruption.\n### **Diagnosis:**\nImpacted customers may have received an error message stating ‘The message violates our Acceptable Use Policy’.\nThe issue has impacted only consumer users and not Google Workspace users.\n### **Workaround:**\nCustomers may retry to send the SMS which might have resolved the issue. Sending SMS messages from Android or iOS devices appeared to be functioning correctly.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-02-17T23:25:55+00:00","modified":"2025-02-17T23:25:56+00:00","when":"2025-02-17T23:25:55+00:00","text":"The issue with Google Voice has been resolved for all affected users as of Monday, 2025-02-17 12:51 US/Pacific.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"gh79SrQk6wqonWoxPdCm","service_name":"Google Voice","affected_products":[{"title":"Google Voice","id":"gh79SrQk6wqonWoxPdCm"}],"uri":"incidents/H8N8DzxWuLAMcZ2Dgjdj"},{"id":"cyCMb3ggSVhQkRC4owxN","number":"1641512534308362385","begin":"2025-02-10T19:50:00+00:00","created":"2025-02-10T20:25:43+00:00","end":"2025-02-10T20:26:00+00:00","modified":"2025-02-14T10:38:02+00:00","external_desc":"**Summary:**\nGoogle Workspace users are experiencing elevated error rates interacting with Google Chat, Google Calendar, Gmail, and Google Meet.\n**Description:**\nWe are experiencing an intermittent issue with Google Chat, Google Calendar, Gmail, Google Meet beginning at Monday, 2025-02-10 11:51 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-02-10 13:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nAffected customers are experiencing intermittent errors interacting with Gmail, Google Calendar, Google Chat, and Google Meet.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-02-13T14:59:00+00:00","modified":"2025-02-14T10:38:02+00:00","when":"2025-02-13T14:59:00+00:00","text":"# Incident Report\n## Summary\nOn Monday, February 10, 2025, Google services including Chat, Calendar, Gmail, Meet, and Docs experienced intermittent 'Error 500' messages and increased latency. The Google Cloud Console also encountered '500' errors and failed to load traffic in South America. These disruptions lasted for 36 minutes and primarily affected users in North and South America regions.\nThis is not the level of quality and reliability we strive to offer you and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nDuring a routine disaster recovery exercise for Google Account Infrastructure, a set of cluster resources in one of our Data Centers were drained, caused by a config change resulting in traffic being redirected to another service in another Data Center. The increased traffic load overwhelmed the service which provides authentication and other core services to Workspace products, leading to errors and increased latencies impacting traffic in North and South America regions. Engineers undrained the set of cluster resources to mitigate the issue.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via internal monitoring on Monday, 10 February at 11:48 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear at 12:12 US/Pacific, Google engineers quickly undrained the set of cluster resources which caused the traffic shifts and overloaded tasks to mitigate the issue at 12:26 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions: - We are working on adding automated guardrails around drain/undrain operations for the relevant services to prevent configuration changes of this type for the relevant services.\n- We are making proactive adjustments to how systems handle network changes to avoid/prevent any potential disruptions.\n- We are addressing a safely draining process in specific areas to ensure smooth and reliable operation and develop a workaround that will allow us to safely manage our network while maintaining uninterrupted service.\n## Detailed Description of Impact\nOn Monday, 10 February 2025 from 11:50 to 12:26 US/Pacific multiple Google Workspace products experienced intermittent '500' errors along with increased latency, and the Google Cloud Console experienced '500' errors and failed to load for traffic for some users in South America.\n### Google Workspace:\n* Google Chat - 65k affected users.\n* Google Calendar - 25k affected users\n* Gmail - 77k affected users\n* Google Meet - 8k affected users\n* Google Docs - 1k affected users\t### Google Cloud Console:\n* Approximately 8k users were affected, predominantly in South America, during the outage. In North America the effect was very short lived (\u003c2 minutes).\nErrors and slowness for a portion of traffic in the Data Center were observed due to overload and cascading failure. Errors led to intermittent 500 errors on all authenticated apps. Some of the issues may have been \"sticky\" for users who were triply affected by broken tasks, leading to more consistent 500 errors, for about 10 minutes.","status":"AVAILABLE"},{"created":"2025-02-11T10:01:03+00:00","modified":"2025-02-13T14:59:00+00:00","when":"2025-02-11T10:01:03+00:00","text":"## Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using **_https://cloud.google.com/support_** or to Google Workspace Support using help article **https://support.google.com/a/answer/1047213**.\n(All Times US/Pacific)\n**Incident Start:** 10 February, 2025 11:50\n**Incident End:** 10 February, 2025 12:26\n**Duration:** 36 minutes\n**Affected Services and Features:**\n**Google Workspace Services:**\nGoogle Chat, Google Calendar, Gmail, Google Meet, Google Docs\n**Google Cloud Services:**\nGoogle Cloud Console\n**Regions/Zones:** Global\n**Description:**\nMultiple Google Workspace products experienced intermittent '500' errors along with increased latency, and the Google Cloud Console experienced '500' errors and failed to load for traffic in South America.\nThe impact lasted for a duration of 36 minutes. From the preliminary analysis the root cause of the issue was temporary overload in a single region. A full Incident Report will be published in the coming days with a detailed root cause analysis.\n**Customer Impact:**\nGoogle Workspace: Multiple products experienced ‘500’ errors or increased latency. The figures below represent the overall number of our customers affected globally over the course of the outage.\n* Google Chat - 65k affected users.\n* Google Calendar - 25k affected users\n* Gmail - 77k affected users\n* Google Meet - 8k affected users\n* Google Docs - 1k affected users\nGoogle Cloud Console:\n* Around 70% of attempts to load the Cloud Console experienced failures or elevated latency in South America for the duration of the outage. Additionally, users in North America also experienced a small blip (\u003c2mins), but this resolved quickly.","status":"AVAILABLE"},{"created":"2025-02-10T21:11:41+00:00","modified":"2025-02-11T10:01:03+00:00","when":"2025-02-10T21:11:41+00:00","text":"The problem with Gmail, Google Calendar, Google Chat, and Google Meet has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-02-10T20:46:10+00:00","modified":"2025-02-10T21:11:41+00:00","when":"2025-02-10T20:46:10+00:00","text":"**Summary:**\nGoogle Workspace users were experiencing elevated error rates interacting with Google Chat, Google Calendar, Gmail, and Google Meet.\n**Description:**\nOur internal metrics show service recovery for all impacted products. Our engineers are continuing to closely monitor for full recovery while investigating the cause of the issue.\nWe will provide an update by Monday, 2025-02-10 14:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nAffected customers were experiencing intermittent errors interacting with Gmail, Google Calendar, Google Chat, and Google Meet.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-02-10T20:25:43+00:00","modified":"2025-02-10T20:46:10+00:00","when":"2025-02-10T20:25:43+00:00","text":"**Summary:**\nGoogle Workspace users are experiencing elevated error rates interacting with Google Chat, Google Calendar, Gmail, and Google Meet.\n**Description:**\nWe are experiencing an intermittent issue with Google Chat, Google Calendar, Gmail, Google Meet beginning at Monday, 2025-02-10 11:51 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-02-10 13:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nAffected customers are experiencing intermittent errors interacting with Gmail, Google Calendar, Google Chat, and Google Meet.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-02-13T14:59:00+00:00","modified":"2025-02-14T10:38:02+00:00","when":"2025-02-13T14:59:00+00:00","text":"# Incident Report\n## Summary\nOn Monday, February 10, 2025, Google services including Chat, Calendar, Gmail, Meet, and Docs experienced intermittent 'Error 500' messages and increased latency. The Google Cloud Console also encountered '500' errors and failed to load traffic in South America. These disruptions lasted for 36 minutes and primarily affected users in North and South America regions.\nThis is not the level of quality and reliability we strive to offer you and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nDuring a routine disaster recovery exercise for Google Account Infrastructure, a set of cluster resources in one of our Data Centers were drained, caused by a config change resulting in traffic being redirected to another service in another Data Center. The increased traffic load overwhelmed the service which provides authentication and other core services to Workspace products, leading to errors and increased latencies impacting traffic in North and South America regions. Engineers undrained the set of cluster resources to mitigate the issue.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via internal monitoring on Monday, 10 February at 11:48 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear at 12:12 US/Pacific, Google engineers quickly undrained the set of cluster resources which caused the traffic shifts and overloaded tasks to mitigate the issue at 12:26 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions: - We are working on adding automated guardrails around drain/undrain operations for the relevant services to prevent configuration changes of this type for the relevant services.\n- We are making proactive adjustments to how systems handle network changes to avoid/prevent any potential disruptions.\n- We are addressing a safely draining process in specific areas to ensure smooth and reliable operation and develop a workaround that will allow us to safely manage our network while maintaining uninterrupted service.\n## Detailed Description of Impact\nOn Monday, 10 February 2025 from 11:50 to 12:26 US/Pacific multiple Google Workspace products experienced intermittent '500' errors along with increased latency, and the Google Cloud Console experienced '500' errors and failed to load for traffic for some users in South America.\n### Google Workspace:\n* Google Chat - 65k affected users.\n* Google Calendar - 25k affected users\n* Gmail - 77k affected users\n* Google Meet - 8k affected users\n* Google Docs - 1k affected users\t### Google Cloud Console:\n* Approximately 8k users were affected, predominantly in South America, during the outage. In North America the effect was very short lived (\u003c2 minutes).\nErrors and slowness for a portion of traffic in the Data Center were observed due to overload and cascading failure. Errors led to intermittent 500 errors on all authenticated apps. Some of the issues may have been \"sticky\" for users who were triply affected by broken tasks, leading to more consistent 500 errors, for about 10 minutes.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/cyCMb3ggSVhQkRC4owxN"},{"id":"8VCWK4rfLd4RjWXDsPdn","number":"10993520394883641254","begin":"2025-02-06T16:52:00+00:00","created":"2025-02-06T17:58:24+00:00","end":"2025-02-06T21:29:00+00:00","modified":"2025-09-04T09:26:47+00:00","external_desc":"**Description:**\nWe are experiencing an issue with Gemini beginning at Wednesday, 2025-02-05 14:11 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-02-06 11:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome Gemini customers are experiencing issues with \"Add File\" or \"Link File\" functionality within Gems\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-02-06T21:49:32+00:00","modified":"2025-09-04T09:02:09+00:00","when":"2025-02-06T21:49:32+00:00","text":"The issue with Gemini has been resolved for all affected users as of Thursday, 2025-02-06 13:29 US/Pacific.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-02-06T21:11:24+00:00","modified":"2025-02-06T21:49:32+00:00","when":"2025-02-06T21:11:24+00:00","text":"**Description:**\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Thursday, 2025-02-06 13:30 US/Pacific.\nWe will provide more information by Thursday, 2025-02-06 14:00 US/Pacific.\n**Symptoms:**\nSome Gemini customers are experiencing issues with \"Add File\" or \"Link File\" functionality within Gems.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-02-06T19:13:48+00:00","modified":"2025-02-06T21:11:24+00:00","when":"2025-02-06T19:13:48+00:00","text":"**Description:**\nWe are experiencing an issue with Gemini beginning on Wednesday, 2025-02-05 14:11 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-02-06 13:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome Gemini customers are experiencing issues with \"Add File\" or \"Link File\" functionality within Gems.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-02-06T17:58:24+00:00","modified":"2025-02-06T19:13:48+00:00","when":"2025-02-06T17:58:24+00:00","text":"**Description:**\nWe are experiencing an issue with Gemini beginning at Wednesday, 2025-02-05 14:11 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-02-06 11:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome Gemini customers are experiencing issues with \"Add File\" or \"Link File\" functionality within Gems\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-02-06T21:49:32+00:00","modified":"2025-09-04T09:02:09+00:00","when":"2025-02-06T21:49:32+00:00","text":"The issue with Gemini has been resolved for all affected users as of Thursday, 2025-02-06 13:29 US/Pacific.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"npdyhgECDJ6tB66MxXyo","service_name":"Gemini","affected_products":[{"title":"Gemini","id":"npdyhgECDJ6tB66MxXyo"}],"uri":"incidents/8VCWK4rfLd4RjWXDsPdn"},{"id":"GzAY5HvYqA9MhjB9ZkSL","number":"562120651607091255","begin":"2025-01-28T15:26:00+00:00","created":"2025-01-28T16:31:00+00:00","end":"2025-01-28T17:34:00+00:00","modified":"2025-02-03T08:51:54+00:00","external_desc":"We're investigating reports of an issue with Admin Console. We will provide more information shortly.","updates":[{"created":"2025-02-03T08:28:43+00:00","modified":"2025-02-03T08:51:54+00:00","when":"2025-02-03T08:28:43+00:00","text":"# Incident Report\n## Summary\nOn Tuesday, 28 January, 2025 at 07:26 US/Pacific, the Admin Console and Cloud Support Platform experienced elevated latency and ‘500’ errors for a duration of 2 hours and 8 minutes. We apologize to our customers affected by the recent service interruption. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nIn December 2024, Google engineers introduced a new internal service to facilitate access to customer metadata as part of storage consolidation efforts. As part of this change, we made configuration changes to read APIs for customer data which were implemented in stages to fully adopt this new service. These read APIs act as a canonical source of truth for validating the users. However, on Tuesday, 28 January, 2025, this service experienced elevated latency for all read APIs resulting in timeout with deadline exceeded errors thereby resulting in latency on the Admin Console.\nThis was triggered by a high error rate observed in a specific task associated with an internal customer data service. This service provides customer data to the system hosting the internal read APIs. The root cause of the issue was identified as the following factors:\n* Increased latency on ‘read’ operations from the data storage system.\n* High latency in internal communication due to the increased time spent waiting for the affected task to be processed.\nDue to these factors, processing threads were unavailable to handle incoming requests quickly, creating a backlog and ultimately leading to timeouts, and errors.\nThe resulting impact was elevated error rates and latency for this internal read API, causing downstream service disruptions and impacting availability for Administrators, the Admin Console.\nThe impact also cascaded to Workspace Customer Support, resulting in approximately 10% of customers being unable to create support cases and initiate chats.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via our internal monitoring system on Tuesday, 28 January, 2025, at 07:26 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear, Google engineers re-directed traffic away from the impacted metadata service to an alternate service. Because both services utilized the same underlying data, there were no issues with respect to data consistency.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n* Perform configuration changes to improve the performance of the service.\n* Add a mandatory check of server’s configuration for every change made to production to ensure high quality of service before going live.\n## Detailed Description of Impact\nOn 28 January, 2025 from 07:26 to 09:34 US/Pacific:\n### Workspace Admin Console and Help Assistant\n* Some customers experienced elevated latency and ‘500’ errors when accessing the Admin console.\n* Some customers could access Admin Console normally, but experienced issues launching ‘Help Assistant’.\n### Workspace Support\n* Some customers experienced issues attempting to open new support cases via Workspace support portal and chat.\n* Customers that had issues with opening cases via the support portal were redirected to our backup Contact Us Form (CUF).\n* Customers that had issues opening cases via chat were redirected to the support portal to create email based cases.","status":"AVAILABLE"},{"created":"2025-01-29T00:59:18+00:00","modified":"2025-02-03T08:28:43+00:00","when":"2025-01-29T00:59:18+00:00","text":"## Mini Incident Report\nWe apologize for the inconvenience this service outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article **[https://support.google.com/a/answer/1047213](https://support.google.com/a/answer/1047213)**.\n(All Times US/Pacific)\n**Incident Start:** 28 January 2025, 07:26\n**Incident End:** 28 January 2025, 09:34\n**Duration:** 2 hours, 8 minutes\n**Affected Services and Features:**\nWorkspace Admin Console and Help Assistant\nWorkspace Support\n**Regions/Zones:** Global\n**Description:**\nWorkspace Admin Console and Workspace Support experienced availability issues intermittently for a duration of 2 hours, 8 minutes. Based on preliminary investigation, the incident was caused by an overloaded metadata service that led to issues with fetching the customer metadata.\nThe issue was mitigated by re-directing traffic away from the impacted metadata service to an alternate service. The traffic will be served from the alternate metadata service until we complete the root cause analysis and fix the underlying issue.\nGoogle will complete a full investigation in the following days that will provide a full root cause.\n**Customer Impact:**\n**Workspace Admin Console and Help Assistant:**\n* Some customers experienced elevated latency and ‘500’ errors when accessing the Admin console.\n* Some customers could access Admin Console normally, but experienced issues launching ‘Help Assistant’.\n**Workspace Support:**\n* Some customers experienced issues attempting to open new support cases via Workspace support portal and chat.\n* Customers that had issues with opening cases via the support portal were redirected to our backup Contact Us Form (CUF).\n* Customers that had issues opening cases via chat were redirected to the support portal to create email based cases.","status":"AVAILABLE"},{"created":"2025-01-28T17:46:12+00:00","modified":"2025-01-29T00:59:18+00:00","when":"2025-01-28T17:46:12+00:00","text":"## Summary:\nAdmin Console may cause slowness or errors for impacted users globally.\n## Diagnosis:\nImpacted users may have experienced slowness or seeing 500 errors while accessing Admin console.\n## Closing Message:\nThe issue with Admin Console has been resolved for all affected users as of Tuesday, 2025-01-28 09:40 US/Pacific. We will publish an analysis of this incident once we have completed our internal investigation. We thank you for your patience while we worked on resolving the issue.","status":"SERVICE_OUTAGE"},{"created":"2025-01-28T17:35:37+00:00","modified":"2025-01-28T17:46:12+00:00","when":"2025-01-28T17:35:37+00:00","text":"## Summary:\nAdmin Console may cause slowness or errors for impacted users globally.\n## Description:\nWe are experiencing an issue with Admin Console beginning on Tuesday, 2025-01-28 07:48 US/Pacific.\nOur engineering team has identified the suspected root cause and is working on mitigation.\nWe will provide an update by Tuesday, 2025-01-28 11:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n## Diagnosis:\nImpacted users may experience slowness or seeing 500 errors while accessing Admin console.\n## Workaround:\nNone at this time.","status":"SERVICE_OUTAGE"},{"created":"2025-01-28T16:31:00+00:00","modified":"2025-01-28T17:35:37+00:00","when":"2025-01-28T16:31:00+00:00","text":"We're investigating reports of an issue with Admin Console. We will provide more information shortly. Impacted users may experience slowness or seeing 500 errors while accessing Admin console.\nWorkaround:\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-02-03T08:28:43+00:00","modified":"2025-02-03T08:51:54+00:00","when":"2025-02-03T08:28:43+00:00","text":"# Incident Report\n## Summary\nOn Tuesday, 28 January, 2025 at 07:26 US/Pacific, the Admin Console and Cloud Support Platform experienced elevated latency and ‘500’ errors for a duration of 2 hours and 8 minutes. We apologize to our customers affected by the recent service interruption. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nIn December 2024, Google engineers introduced a new internal service to facilitate access to customer metadata as part of storage consolidation efforts. As part of this change, we made configuration changes to read APIs for customer data which were implemented in stages to fully adopt this new service. These read APIs act as a canonical source of truth for validating the users. However, on Tuesday, 28 January, 2025, this service experienced elevated latency for all read APIs resulting in timeout with deadline exceeded errors thereby resulting in latency on the Admin Console.\nThis was triggered by a high error rate observed in a specific task associated with an internal customer data service. This service provides customer data to the system hosting the internal read APIs. The root cause of the issue was identified as the following factors:\n* Increased latency on ‘read’ operations from the data storage system.\n* High latency in internal communication due to the increased time spent waiting for the affected task to be processed.\nDue to these factors, processing threads were unavailable to handle incoming requests quickly, creating a backlog and ultimately leading to timeouts, and errors.\nThe resulting impact was elevated error rates and latency for this internal read API, causing downstream service disruptions and impacting availability for Administrators, the Admin Console.\nThe impact also cascaded to Workspace Customer Support, resulting in approximately 10% of customers being unable to create support cases and initiate chats.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via our internal monitoring system on Tuesday, 28 January, 2025, at 07:26 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear, Google engineers re-directed traffic away from the impacted metadata service to an alternate service. Because both services utilized the same underlying data, there were no issues with respect to data consistency.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n* Perform configuration changes to improve the performance of the service.\n* Add a mandatory check of server’s configuration for every change made to production to ensure high quality of service before going live.\n## Detailed Description of Impact\nOn 28 January, 2025 from 07:26 to 09:34 US/Pacific:\n### Workspace Admin Console and Help Assistant\n* Some customers experienced elevated latency and ‘500’ errors when accessing the Admin console.\n* Some customers could access Admin Console normally, but experienced issues launching ‘Help Assistant’.\n### Workspace Support\n* Some customers experienced issues attempting to open new support cases via Workspace support portal and chat.\n* Customers that had issues with opening cases via the support portal were redirected to our backup Contact Us Form (CUF).\n* Customers that had issues opening cases via chat were redirected to the support portal to create email based cases.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/GzAY5HvYqA9MhjB9ZkSL"},{"id":"FoXARVPND3oeDBu6tLuT","number":"5200787915827209729","begin":"2025-01-28T14:00:00+00:00","created":"2025-01-28T15:34:38+00:00","end":"2025-01-28T17:36:00+00:00","modified":"2025-01-28T18:20:38+00:00","external_desc":"Some Workspace Support customers are experiencing issues with creating customer cases\nWe are experiencing an issue with Workspace Support beginning at Tuesday, 2025-01-28 06:00 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Tuesday, 2025-01-28 08:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\nCustomer Symptoms: Some customers may experience an error with creating customer cases.\nWorkaround: After several unsuccessful attempts the customers would be guided to use the Contact Us Form as a backup.","updates":[{"created":"2025-01-28T18:01:55+00:00","modified":"2025-01-28T18:20:38+00:00","when":"2025-01-28T18:01:55+00:00","text":"The issue with Google Cloud Support has been resolved for all affected users as of Tuesday, 2025-01-28 09:36 US/Pacific.\nOur preliminary analysis shows the issue was triggered by a software rollout and the issue was mitigated by redirecting the traffic away from the impacted clusters and performing a rollback.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-01-28T17:40:42+00:00","modified":"2025-01-28T18:01:55+00:00","when":"2025-01-28T17:40:42+00:00","text":"Mitigation work is completed by our engineering team.\nOur internal monitoring shows that the errors have subsided and we believe that there is no more customer impact. We are continuing to closely monitor to ensure full resolution.\nWe will provide more information by Tuesday, 2025-01-28 10:30 US/Pacific.\nWe apologize to all who are affected by the disruption.\nCustomer Symptoms: Some customers may experience an error with creating customer cases.\nWorkaround: Affected customers should be able to retry successfully.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-28T17:21:42+00:00","modified":"2025-01-28T17:40:42+00:00","when":"2025-01-28T17:21:42+00:00","text":"Some Workspace Support customers are experiencing issues with creating customer cases via Support Portal and Chat\nMitigation work is still underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Tuesday, 2025-01-28 10:30 US/Pacific.\nCustomer Symptoms: Some customers may experience an error with creating customer cases.\nWorkaround: Affected customers should be able to retry successfully.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-28T16:10:02+00:00","modified":"2025-01-28T17:21:42+00:00","when":"2025-01-28T16:10:02+00:00","text":"Some Workspace Support customers are experiencing issues with creating customer cases via Portal and Chat\nWe are experiencing an issue with Workspace Support beginning at Tuesday, 2025-01-28 06:00 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Tuesday, 2025-01-28 09:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\nCustomer Symptoms: Some customers may experience an error with creating customer cases.\nWorkaround: After several unsuccessful attempts the customers would be guided to use the Contact Us Form as a backup.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-28T15:59:30+00:00","modified":"2025-01-28T16:10:02+00:00","when":"2025-01-28T15:59:30+00:00","text":"Some Workspace Support customers are experiencing issues with creating customer cases\nWe are experiencing an issue with Workspace Support beginning at Tuesday, 2025-01-28 06:00 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Tuesday, 2025-01-28 09:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\nCustomer Symptoms: Some customers may experience an error with creating customer cases.\nWorkaround: After several unsuccessful attempts the customers would be guided to use the Contact Us Form as a backup.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-28T15:34:38+00:00","modified":"2025-01-28T15:59:30+00:00","when":"2025-01-28T15:34:38+00:00","text":"Some Workspace Support customers are experiencing issues with creating customer cases\nWe are experiencing an issue with Workspace Support beginning at Tuesday, 2025-01-28 06:00 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Tuesday, 2025-01-28 08:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\nCustomer Symptoms: Some customers may experience an error with creating customer cases.\nWorkaround: After several unsuccessful attempts the customers would be guided to use the Contact Us Form as a backup.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-01-28T18:01:55+00:00","modified":"2025-01-28T18:20:38+00:00","when":"2025-01-28T18:01:55+00:00","text":"The issue with Google Cloud Support has been resolved for all affected users as of Tuesday, 2025-01-28 09:36 US/Pacific.\nOur preliminary analysis shows the issue was triggered by a software rollout and the issue was mitigated by redirecting the traffic away from the impacted clusters and performing a rollback.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/FoXARVPND3oeDBu6tLuT"},{"id":"sNYNFVMFzzZhLdJPxgvU","number":"17763007403616328777","begin":"2025-01-27T17:00:00+00:00","created":"2025-01-29T22:16:40+00:00","end":"2025-01-30T05:00:00+00:00","modified":"2025-01-30T20:59:13+00:00","external_desc":"**Summary:**\nGoogle Drive, Gmail Web, - Unable to see PDF annotations.\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-01-29 13:24 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-01-29 15:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nCustomers using Drive Web, Gmail Web, opening PDFs would not see PDF Annotations (shapes, user added text, freehand drawing, etc). All PDF data is still available.\n**Workaround:**\nOpen the PDF in a different tool.","updates":[{"created":"2025-01-30T20:58:25+00:00","modified":"2025-01-30T20:59:13+00:00","when":"2025-01-30T20:58:25+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\nIncident Start: 27 Jan 2025 09:00\nIncident End: 29 Jan 2025 21:00\nDuration: 2 days, 12 hours\n**Affected Services and Features:**\nGoogle Drive, Gmail\n**Regions/Zones:** Global\n**Description:**\nUsers experienced an issue where PDF annotations were not displayed when using Google Drive Web and Gmail Web for a period of 2 days, 12 hours. All PDF data was still available if downloaded.\nA preliminary analysis indicates the incident was caused by a code change that incorrectly determined which portions of a PDF to display.\nGoogle engineers mitigated the issue by rolling back the affected client builds and deployed a preventative server fix to ensure this incident does not occur again.\n**Customer Impact:**\nAffected users were unable to see PDF annotations, such as filled form fields, shapes, freehand drawing, in the previewer.","status":"AVAILABLE"},{"created":"2025-01-30T03:25:35+00:00","modified":"2025-01-30T20:58:25+00:00","when":"2025-01-30T03:25:35+00:00","text":"**Summary:**\nGoogle Drive, Gmail Web, - Unable to see PDF annotations.\n**Closing Message:**\nThe issue with Google Drive, Gmail has been resolved for all affected users as of Wednesday, 2025-01-29 18:55 US/Pacific.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-01-30T00:37:19+00:00","modified":"2025-01-30T03:25:35+00:00","when":"2025-01-30T00:37:19+00:00","text":"**Summary:**\nGoogle Drive, Gmail Web, - Unable to see PDF annotations.\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-01-29 21:30 US/Pacific.\n**Diagnosis:**\nGoogle Drive Web and Gmail Web customers when opening PDFs would not see PDF Annotations (shapes, user added text, freehand drawing, etc). All PDF data is still available.\n**Workaround:**\nDownload the PDF file and open them locally.","status":"SERVICE_INFORMATION"},{"created":"2025-01-29T22:57:20+00:00","modified":"2025-01-30T00:37:19+00:00","when":"2025-01-29T22:57:20+00:00","text":"**Summary:**\nGoogle Drive, Gmail Web, - Unable to see PDF annotations.\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-01-29 13:48 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-01-29 17:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nCustomers using Drive Web and Gmail Web opening PDFs would not see PDF Annotations (shapes, user added text, freehand drawing, etc). All PDF data is still available.\n**Workaround:**\nDownload the PDF file and open them locally.","status":"SERVICE_INFORMATION"},{"created":"2025-01-29T22:16:40+00:00","modified":"2025-01-29T22:57:20+00:00","when":"2025-01-29T22:16:40+00:00","text":"**Summary:**\nGoogle Drive, Gmail Web, - Unable to see PDF annotations.\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-01-29 13:24 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-01-29 15:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nCustomers using Drive Web, Gmail Web, opening PDFs would not see PDF Annotations (shapes, user added text, freehand drawing, etc). All PDF data is still available.\n**Workaround:**\nOpen the PDF in a different tool.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-01-30T20:58:25+00:00","modified":"2025-01-30T20:59:13+00:00","when":"2025-01-30T20:58:25+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\n(All Times US/Pacific)\nIncident Start: 27 Jan 2025 09:00\nIncident End: 29 Jan 2025 21:00\nDuration: 2 days, 12 hours\n**Affected Services and Features:**\nGoogle Drive, Gmail\n**Regions/Zones:** Global\n**Description:**\nUsers experienced an issue where PDF annotations were not displayed when using Google Drive Web and Gmail Web for a period of 2 days, 12 hours. All PDF data was still available if downloaded.\nA preliminary analysis indicates the incident was caused by a code change that incorrectly determined which portions of a PDF to display.\nGoogle engineers mitigated the issue by rolling back the affected client builds and deployed a preventative server fix to ensure this incident does not occur again.\n**Customer Impact:**\nAffected users were unable to see PDF annotations, such as filled form fields, shapes, freehand drawing, in the previewer.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/sNYNFVMFzzZhLdJPxgvU"},{"id":"n3bN6Xsdr4gmPxz29x2N","number":"4601265069960654613","begin":"2025-01-23T22:30:00+00:00","created":"2025-01-24T02:03:01+00:00","end":"2025-01-25T01:30:00+00:00","modified":"2025-01-25T01:49:34+00:00","external_desc":"### **Summary:**\nSome Gmail Android users may experience elevated crashes\n### **Description:**\nWe are experiencing an issue with Gmail beginning on Thursday, 2025-01-23 14:30 US/Pacific.\nOur engineering team is investigating the issue.\nWe will provide an update by Thursday, 2025-01-23 19:30 US/Pacific.\nWe apologize to all affected by the disruption.\n### **Diagnosis:**\nCustomers accessing non-gmail accounts via the gmail app on android devices may experience crashes while opening the email.\n## **Workaround:**\nNone at this time.","updates":[{"created":"2025-01-25T01:49:34+00:00","modified":"2025-01-25T01:49:34+00:00","when":"2025-01-25T01:49:34+00:00","text":"### **Description:**\nThe issue with Gmail has been resolved for all affected users as of Friday, 2025-01-24 17:30 US/Pacific. We thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-01-24T23:07:32+00:00","modified":"2025-01-25T01:49:34+00:00","when":"2025-01-24T23:07:32+00:00","text":"### **Summary:**\nSome Gmail Android users may experience elevated crashes\n### **Description:**\nOur Engineering team has completed a rollout intended to mitigate the issue. We continue to observe a decrease in the crash rate and will continue to monitor for the stability.\nWe will provide more information by Monday, 2025-01-27 09:00 US/Pacific.\n### **Diagnosis:**\nCustomers accessing non-gmail accounts via the gmail app on android devices may experience crashes while opening the email.\n## **Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-24T07:14:40+00:00","modified":"2025-01-24T23:07:32+00:00","when":"2025-01-24T07:14:40+00:00","text":"### **Summary:**\nSome Gmail Android users may experience elevated crashes\n### **Description:**\nOur Engineering team has completed a rollout intended to mitigate the issue. We are observing a decrease in the crash rate and will continue to monitor for the stability.\nWe will provide more information by Friday, 2025-01-24 09:30 US/Pacific.\n### **Diagnosis:**\nCustomers accessing non-gmail accounts via the gmail app on android devices may experience crashes while opening the email.\n## **Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-24T03:27:42+00:00","modified":"2025-01-24T07:14:40+00:00","when":"2025-01-24T03:27:42+00:00","text":"### **Summary:**\nSome Gmail Android users may experience elevated crashes\n### **Description:**\nOur Engineering team has completed a rollout intended to mitigate the issue. We are observing a decrease in the crash rate and will continue to monitor for the stability.\nWe will provide more information by Thursday, 2025-01-23 23:30 US/Pacific.\n### **Diagnosis:**\nCustomers accessing non-gmail accounts via the gmail app on android devices may experience crashes while opening the email.\n## **Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-01-24T02:03:01+00:00","modified":"2025-01-24T03:27:42+00:00","when":"2025-01-24T02:03:01+00:00","text":"### **Summary:**\nSome Gmail Android users may experience elevated crashes\n### **Description:**\nWe are experiencing an issue with Gmail beginning on Thursday, 2025-01-23 14:30 US/Pacific.\nOur engineering team is investigating the issue.\nWe will provide an update by Thursday, 2025-01-23 19:30 US/Pacific.\nWe apologize to all affected by the disruption.\n### **Diagnosis:**\nCustomers accessing non-gmail accounts via the gmail app on android devices may experience crashes while opening the email.\n## **Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-01-25T01:49:34+00:00","modified":"2025-01-25T01:49:34+00:00","when":"2025-01-25T01:49:34+00:00","text":"### **Description:**\nThe issue with Gmail has been resolved for all affected users as of Friday, 2025-01-24 17:30 US/Pacific. We thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/n3bN6Xsdr4gmPxz29x2N"},{"id":"UTBmhHDLSGFN375e2t2j","number":"10020479171617099643","begin":"2025-01-18T02:00:00+00:00","created":"2025-01-22T18:59:23+00:00","end":"2025-01-22T19:10:00+00:00","modified":"2025-01-22T20:12:28+00:00","external_desc":"Customers attempting to start a direct message chat with a new contact from the Chat Web search bar may receive the error “Can’t start a direct message right now. Try again later.”\nCustomers can attempt to start a new direct message using the \"New Chat\" button.\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-01-22 12:00 US/Pacific.","updates":[{"created":"2025-01-22T20:12:00+00:00","modified":"2025-01-22T20:12:28+00:00","when":"2025-01-22T20:12:00+00:00","text":"The problem with Google Chat has been resolved. We apologize for the inconvenience and thank you for your patience and continued support. Customers that are still experiencing errors when starting a direct message from the Chat Web search bar should refresh their browser page. Customers can attempt to start a new direct message using the \"New Chat\" button.","status":"AVAILABLE"},{"created":"2025-01-22T19:31:33+00:00","modified":"2025-01-22T20:12:00+00:00","when":"2025-01-22T19:31:33+00:00","text":"Our engineers have implemented a mitigation for this issue and are currently validating.\nCustomers that are still experiencing errors when starting a direct message from Chat Web search bar should refresh their browser page.\nWe will provide more information by Wednesday, 2025-01-22 12:30 US/Pacific.","status":"SERVICE_INFORMATION"},{"created":"2025-01-22T18:59:23+00:00","modified":"2025-01-22T19:31:33+00:00","when":"2025-01-22T18:59:23+00:00","text":"Customers attempting to start a direct message chat with a new contact from the Chat Web search bar may receive the error “Can’t start a direct message right now. Try again later.”\nCustomers can attempt to start a new direct message using the \"New Chat\" button.\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-01-22 12:00 US/Pacific.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-01-22T20:12:00+00:00","modified":"2025-01-22T20:12:28+00:00","when":"2025-01-22T20:12:00+00:00","text":"The problem with Google Chat has been resolved. We apologize for the inconvenience and thank you for your patience and continued support. Customers that are still experiencing errors when starting a direct message from the Chat Web search bar should refresh their browser page. Customers can attempt to start a new direct message using the \"New Chat\" button.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"Ht9Vx5PFzDPHYvrnrvSk","service_name":"Google Chat","affected_products":[{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/UTBmhHDLSGFN375e2t2j"}]
<!--
Mirrored from: https://www.google.com/appsstatus/dashboard/incidents.json
Mirrored at: 2025-12-19 08:05:24 UTC
Mirror URL: https://google.hw233.vip/appsstatus/dashboard/incidents.json
-->