Error="failed to update rule group: invalid character '_' after object key:value pair"

  • What Grafana version and what operating system are you using?
    8.3.6

  • What are you trying to achieve?
    create alerting rule

  • How are you trying to achieve it?
    create alerting rule

  • What happened?

Failed to save rule: failed to update rule group: invalid character ‘_’ after object key:value pair

error occurs

  • What did you expect to happen?

alerting rule created

  • Can you copy/paste the configuration(s) that you are having problems with?
    |DEFAULT||
    |—|—|
    |app_mode|production|
    |instance_name|grafana-8ff7f5dc5-f5dwl|
    |alerting||
    |concurrent_render_limit|5|
    |enabled||
    |error_or_timeout|alerting|
    |evaluation_timeout_seconds|30|
    |execute_alerts|true|
    |max_annotation_age||
    |max_annotations_to_keep|0|
    |max_attempts|3|
    |min_interval_seconds|1|
    |nodata_or_nullvalues|no_data|
    |notification_timeout_seconds|30|
    |analytics||
    |application_insights_connection_string||
    |application_insights_endpoint_url||
    |check_for_updates|true|
    |google_analytics_ua_id||
    |google_tag_manager_id||
    |reporting_distributor|grafana-labs|
    |reporting_enabled|true|
    |rudderstack_data_plane_url||
    |rudderstack_write_key||
    |annotations||
    |cleanupjob_batchsize|100|
    |annotations.api||
    |max_age|2M|
    |max_annotations_to_keep|50000|
    |annotations.dashboard||
    |max_age|2M|
    |max_annotations_to_keep|50000|
    |auth||
    |api_key_max_seconds_to_live|-1|
    |disable_login_form|false|
    |disable_signout_menu|false|
    |login_cookie_name|grafana_session|
    |login_maximum_inactive_lifetime_days||
    |login_maximum_inactive_lifetime_duration|7d|
    |login_maximum_lifetime_days||
    |login_maximum_lifetime_duration|30d|
    |oauth_auto_login|false|
    |oauth_state_cookie_max_age|600|
    |signout_redirect_url||
    |sigv4_auth_enabled|false|
    |token_rotation_interval_minutes|10|
    |auth.anonymous||
    |enabled|false|
    |hide_version|false|
    |org_name|Main Org.|
    |org_role|Viewer|
    |auth.azuread||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_groups||
    |api_url||
    |auth_url||
    |client_id|some_client_id|
    |client_secret|
    |
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |name|Azure AD|
    |role_attribute_path||
    |role_attribute_strict||
    |scopes|openid email profile|
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url|
    |
    |use_pkce||
    |auth.basic||
    |enabled|true|
    |auth.generic_oauth||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_organizations||
    |api_url||
    |auth_url||
    |client_id|some_id|
    |client_secret|
    |
    |email_attribute_name|email:primary|
    |email_attribute_path||
    |empty_scopes|false|
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |id_token_attribute_name||
    |login_attribute_path||
    |name|OAuth|
    |name_attribute_path||
    |role_attribute_path||
    |role_attribute_strict|false|
    |scopes|user:email|
    |team_ids||
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure|false|
    |token_url||
    |use_pkce|false|
    |auth.github||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_organizations||
    |api_url|
    |
    |auth_url|
    |
    |client_id|some_id|
    |client_secret||
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |name|github|
    |role_attribute_path||
    |role_attribute_strict||
    |scopes|user:email,read:org|
    |team_ids||
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url|
    |
    |use_pkce||
    |auth.gitlab||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_groups||
    |api_url|
    |
    |auth_url|
    |
    |client_id|some_id|
    |client_secret||
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |name|gitlab|
    |role_attribute_path||
    |role_attribute_strict||
    |scopes|api|
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url|
    |
    |use_pkce||
    |auth.google||
    |allow_sign_up|true|
    |allowed_domains||
    |api_url|
    |
    |auth_url|
    |
    |client_id|some_client_id|
    |client_secret||
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |name|google|
    |role_attribute_path||
    |role_attribute_strict||
    |scopes|
    |
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url|
    |
    |use_pkce||
    |auth.grafana_com||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_organizations||
    |api_url||
    |auth_url||
    |client_id|some_id|
    |client_secret|
    |
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |name|grafana_com|
    |role_attribute_path||
    |role_attribute_strict||
    |scopes|user:email|
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url||
    |use_pkce||
    |auth.grafananet||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_organizations||
    |api_url||
    |auth_url||
    |client_id|some_id|
    |client_secret|
    |
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|false|
    |groups_attribute_path||
    |hosted_domain||
    |name|grafananet|
    |role_attribute_path||
    |role_attribute_strict||
    |scopes|user:email|
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url||
    |use_pkce||
    |auth.jwt||
    |cache_ttl|60m|
    |email_claim||
    |enabled|false|
    |expect_claims|{}|
    |expected_claims|{}|
    |header_name||
    |jwk_set_file||
    |jwk_set_url||
    |key_file||
    |username_claim||
    |auth.ldap||
    |active_sync_enabled|true|
    |allow_sign_up|true|
    |config_file|/etc/grafana/ldap.toml|
    |enabled|false|
    |sync_cron|0 0 1 * * |
    |auth.okta||
    |allow_sign_up|true|
    |allowed_domains||
    |allowed_groups||
    |api_url||
    |auth_url||
    |client_id||
    |client_secret|
    |
    |email_attribute_name||
    |email_attribute_path||
    |empty_scopes||
    |enabled|true|
    |groups_attribute_path||
    |hosted_domain||
    |name|Okta|
    |role_attribute_path||
    |role_attribute_strict|false|
    |scopes|openid profile email groups|
    |team_ids_attribute_path||
    |teams_url||
    |tls_client_ca||
    |tls_client_cert||
    |tls_client_key||
    |tls_skip_verify_insecure||
    |token_url|
    |
    |use_pkce||
    |auth.proxy||
    |auto_sign_up|true|
    |enable_login_token|false|
    |enabled|false|
    |header_name|X-WEBAUTH-USER|
    |header_property|username|
    |headers||
    |ldap_sync_ttl|60|
    |sync_ttl|60|
    |whitelist||
    |auth.saml||
    |enabled|false|
    |name|SAML|
    |aws||
    |allowed_auth_providers|default,keys,credentials|
    |assume_role_enabled|true|
    |list_metrics_page_limit|500|
    |azure||
    |cloud|AzureCloud|
    |managed_identity_client_id||
    |managed_identity_enabled|false|
    |caching||
    |enabled|true|
    |dashboards||
    |default_home_dashboard_path||
    |min_refresh_interval|5s|
    |versions_to_keep|20|
    |database||
    |ca_cert_path||
    |cache_mode|private|
    |client_cert_path||
    |client_key_path||
    |conn_max_lifetime|14400|
    |connection_string||
    |host|gcloud-sqlproxy.infra.svc:1478|
    |isolation_level||
    |log_queries|false|
    |max_idle_conn|2|
    |max_open_conn|0|
    |name|grafana|
    |password|
    |
    |path|grafana.db|
    |server_cert_name||
    |skip_migrations||
    |ssl_mode|disable|
    |type|mysql|
    |url||
    |user|grafana|
    |dataproxy||
    |dialTimeout|10|
    |expect_continue_timeout_seconds|1|
    |idle_conn_timeout_seconds|90|
    |keep_alive_seconds|30|
    |logging|false|
    |max_conns_per_host|0|
    |max_idle_connections|100|
    |max_idle_connections_per_host||
    |response_limit|0|
    |row_limit|1000000|
    |send_user_header|false|
    |timeout|30|
    |tls_handshake_timeout_seconds|10|
    |datasources||
    |datasource_limit|5000|
    |date_formats||
    |default_timezone|browser|
    |default_week_start|browser|
    |full_date|YYYY-MM-DD HH:mm:ss|
    |interval_day|MM/DD|
    |interval_hour|MM/DD HH:mm|
    |interval_minute|HH:mm|
    |interval_month|YYYY-MM|
    |interval_second|HH:mm:ss|
    |interval_year|YYYY|
    |use_browser_locale|false|
    |emails||
    |content_types|text/html|
    |templates_pattern|emails/
    .html,emails/
    .txt|
    |welcome_email_on_sign_up|false|
    |enterprise||
    |license_path|/var/lib/grafana/license.jwt|
    |explore||
    |enabled|true|
    |expressions||
    |enabled|true|
    |external_image_storage||
    |provider||
    |external_image_storage.azure_blob||
    |account_key|
    |
    |account_name||
    |container_name||
    |external_image_storage.gcs||
    |bucket||
    |enable_signed_urls|false|
    |key_file||
    |path||
    |signed_url_expiration||
    |external_image_storage.local||
    |external_image_storage.s3||
    |access_key||
    |bucket||
    |bucket_url||
    |endpoint||
    |path||
    |path_style_access||
    |region||
    |secret_key|
    |
    |external_image_storage.webdav||
    |password|
    |
    |public_url||
    |url||
    |username||
    |feature_toggles||
    |enable||
    |geomap||
    |default_baselayer_config||
    |enable_custom_baselayers|true|
    |grafana_com||
    |url|
    |
    |grafana_net||
    |url|
    |
    |live||
    |allowed_origins||
    |ha_engine||
    |ha_engine_address|127.0.0.1:6379|
    |max_connections|100|
    |log||
    |filters||
    |level|info|
    |mode|console|
    |log.console||
    |format|console|
    |level|info|
    |log.file||
    |daily_rotate|true|
    |format|text|
    |level||
    |log_rotate|true|
    |max_days|7|
    |max_lines|1000000|
    |max_size_shift|28|
    |log.frontend||
    |custom_endpoint|/log|
    |enabled|false|
    |log_endpoint_burst_limit|15|
    |log_endpoint_requests_per_second_limit|3|
    |sample_rate|1.0|
    |sentry_dsn||
    |log.syslog||
    |address||
    |facility||
    |format|text|
    |level||
    |network||
    |tag||
    |metrics||
    |basic_auth_password|
    |
    |basic_auth_username||
    |disable_total_stats|false|
    |enabled|true|
    |interval_seconds|10|
    |metrics.environment_info||
    |metrics.graphite||
    |address||
    |prefix|prod.grafana.%(instance_name)s.|
    |panels||
    |disable_sanitize_html|false|
    |enable_alpha|false|
    |paths||
    |data|/var/lib/grafana/|
    |logs|/var/log/grafana|
    |plugins|/var/lib/grafana/plugins|
    |provisioning|/etc/grafana/provisioning|
    |temp_data_lifetime|24h|
    |plugin.grafana-image-renderer||
    |grpc_host||
    |grpc_port||
    |rendering_args||
    |rendering_chrome_bin||
    |rendering_clustering_max_concurrency||
    |rendering_clustering_mode||
    |rendering_clustering_timeout||
    |rendering_dumpio||
    |rendering_ignore_https_errors||
    |rendering_language||
    |rendering_mode||
    |rendering_timezone||
    |rendering_verbose_logging||
    |rendering_viewport_device_scale_factor||
    |rendering_viewport_max_device_scale_factor||
    |rendering_viewport_max_height||
    |rendering_viewport_max_width||
    |plugins||
    |allow_loading_unsigned_plugins||
    |app_tls_skip_verify_insecure|false|
    |enable_alpha|false|
    |plugin_admin_enabled|true|
    |plugin_admin_external_manage_enabled|false|
    |plugin_catalog_hidden_plugins||
    |plugin_catalog_url|
    |
    |quota||
    |enabled|false|
    |global_alert_rule|-1|
    |global_api_key|-1|
    |global_dashboard|-1|
    |global_data_source|-1|
    |global_org|-1|
    |global_session|-1|
    |global_user|-1|
    |org_alert_rule|100|
    |org_api_key|10|
    |org_dashboard|100|
    |org_data_source|10|
    |org_user|10|
    |user_org|10|
    |recorded_queries||
    |enabled|false|
    |remote_cache||
    |connstr||
    |type|database|
    |rendering||
    |callback_url||
    |concurrent_render_request_limit|30|
    |server_url||
    |security||
    |admin_password|
    |
    |admin_user|admin|
    |allow_embedding|false|
    |available_encryption_providers||
    |content_security_policy|false|
    |content_security_policy_template|script-src ‘self’ ‘unsafe-eval’ ‘unsafe-inline’ ‘strict-dynamic’ $NONCE;object-src ‘none’;font-src ‘self’;style-src ‘self’ ‘unsafe-inline’ blob:;img-src * data:;base-uri ‘self’;connect-src ‘self’ grafana.com ws://$ROOT_PATH wss://$ROOT_PATH;manifest-src ‘self’;media-src ‘none’;form-action ‘self’;|
    |cookie_samesite|lax|
    |cookie_secure|false|
    |data_source_proxy_whitelist||
    |disable_brute_force_login_protection|false|
    |disable_gravatar|false|
    |disable_initial_admin_creation|false|
    |encryption_provider|secretKey|
    |secret_key|
    |
    |strict_transport_security|false|
    |strict_transport_security_max_age_seconds|86400|
    |strict_transport_security_preload|false|
    |strict_transport_security_subdomains|false|
    |x_content_type_options|true|
    |x_xss_protection|true|
    |server||
    |cdn_url||
    |cert_file||
    |cert_key||
    |domain|localhost|
    |enable_gzip|false|
    |enforce_domain|false|
    |http_addr|0.0.0.0|
    |http_port|3000|
    |protocol|http|
    |read_timeout|0|
    |root_url|
    |
    |router_logging|false|
    |serve_from_sub_path|false|
    |socket|/tmp/grafana.sock|
    |static_root_path|public|
    |smtp||
    |cert_file||
    |ehlo_identity||
    |enabled|false|
    |from_address|admin@grafana.localhost|
    |from_name|Grafana|
    |host|localhost:25|
    |key_file||
    |password|
    |
    |skip_verify|false|
    |startTLS_policy||
    |user||
    |snapshots||
    |external_enabled|true|
    |external_snapshot_name|Publish to snapshots.raintank.io|
    |external_snapshot_url|
    |
    |public_mode|false|
    |snapshot_remove_expired|true|
    |tracing.jaeger||
    |address||
    |always_included_tag||
    |disable_shared_zipkin_spans|false|
    |sampler_param|1|
    |sampler_type|const|
    |sampling_server_url||
    |zipkin_propagation|false|
    |unified_alerting||
    |admin_config_poll_interval|60s|
    |alertmanager_config_poll_interval|60s|
    |disabled_orgs||
    |enabled||
    |evaluation_timeout|30s|
    |execute_alerts|true|
    |ha_advertise_address||
    |ha_gossip_interval|200ms|
    |ha_listen_address|0.0.0.0:9094|
    |ha_peer_timeout|15s|
    |ha_peers||
    |ha_push_pull_interval|60s|
    |max_attempts|3|
    |min_interval|10s|
    |users||
    |allow_org_create|false|
    |allow_sign_up|false|
    |auto_assign_org|true|
    |auto_assign_org_id|1|
    |auto_assign_org_role|Viewer|
    |default_theme|dark|
    |editors_can_admin|false|
    |external_manage_info||
    |external_manage_link_name||
    |external_manage_link_url||
    |hidden_users||
    |home_page||
    |login_hint|email or username|
    |password_hint|
    ******|
    |user_invite_max_lifetime_duration|24h|
    |verify_email_enabled|false|
    |viewers_can_edit|false|

  • Did you receive any errors in the Grafana UI or in related logs? If so, please tell us exactly what they were.
    t=2022-02-22T15:06:44+0000 lvl=eror msg=“failed to update rule group: invalid character ‘’ after object key:value pair" logger=context userId=8 orgId=1 uname=*** error="failed to update rule group: invalid character '’ afte │
    │ r object key:value pair” remote_addr=10.2.0.175 │
    │ t=2022-02-22T15:06:44+0000 lvl=eror msg=“Request Completed” logger=context userId=8 orgId=1 uname=r.verenich@melsoft-games.com method=GET path=/api/ruler/grafana/api/v1/rules/Preset status=500 remote_addr=10.2.0.175 time_ms=10 size=92 referer=“https: │
    │ {LINK_HERE} │
    │ to%22:0%7D,%22datasourceUid%22:%22kQ9mhoank%22,%22model%22:%7B%22builder%22:%7B%22intervals%22:%7B%22intervals%22:%5B%222022-02-22T09:05:28.352Z%2F2022-02-22T15:05:28.352Z%22%5D,%22type%22:%22intervals%22%7D,%22query%22:%22SELECT%20TIME_FLOOR(CAST(%5 │
    │ C%22__time%5C%22%20AS%20TIMESTAMP),%20%27PT5M%27)%20AS%20%5C%22__timestamp%5C%22,%5Cn%20%20%20%20%20%20%20APPROX_COUNT_DISTINCT_DS_HLL(users_count_distinct)%20AS%20%5C%22distinct%20users%5C%22,%5Cn%20%20%20%20%20%20%20sum(%5C%22count_event_occur%5C%2 │
    │ 2)%20AS%20%5C%22event%20count%5C%22%5CnFROM%20%5C%22druid%5C%22.%5C%22live-tables%5C%22%5CnWHERE%20%5C%22__time%5C%22%20%3E%3D%20CURRENT_TIMESTAMP%20-%20INTERVAL%20%2724%27%20HOUR%5Cn%20%20AND%20%5C%22__time%5C%22%20%3C%20CURRENT_TIMESTAMP%5Cn%20%20A │
    │ ND%20%5C%22event_name%5C%22%20IN%20(%27ads_show%27)%5CnGROUP%20BY%20TIME_FLOOR(CAST(%5C%22__time%5C%22%20AS%20TIMESTAMP),%20%27PT5M%27)%5Cn–ORDER%20BY%20%5C%22HLLSketchMerge%5C%22%20DESC%5CnLIMIT%2050000%22,%22queryType%22:%22sql%22%7D,%22datasource │
    │ %22:%7B%22type%22:%22grafadruid-druid-datasource%22,%22uid%22:%22kQ9mhoank%22%7D,%22refId%22:%22A%22,%22settings%22:%7B%7D,%22expr%22:%22%7B%5C%22builder%5C%22:%7B%5C%22queryType%5C%22:%5C%22sql%5C%22,%5C%22query%5C%22:%5C%22SELECT%20TIME_FLOOR(CAST( │
    │ %5C%5C%5C%22__time%5C%5C%5C%22%20AS%20TIMESTAMP),%20%27PT5M%27)%20AS%20%5C%5C%5C%22__timestamp%5C%5C%5C%22,%5C%5Cn%20%20%20%20%20%20%20APPROX_COUNT_DISTINCT_DS_HLL(users_count_distinct)%20AS%20%5C%5C%5C%22distinct%20users%5C%5C%5C%22,%5C%5Cn%20%20%20 │
    │ %20%20%20%20sum(%5C%5C%5C%22count_event_occur%5C%5C%5C%22)%20AS%20%5C%5C%5C%22event%20count%5C%5C%5C%22%5C%5CnFROM%20%5C%5C%5C%22druid%5C%5C%5C%22.%5C%5C%5C%22live-tables%5C%5C%5C%22%5C%5CnWHERE%20%5C%5C%5C%22__time%5C%5C%5C%22%20%3E%3D%20CURRENT_TIM │
    │ ESTAMP%20-%20INTERVAL%20%2724%27%20HOUR%5C%5Cn%20%20AND%20%5C%5C%5C%22__time%5C%5C%5C%22%20%3C%20CURRENT_TIMESTAMP%5C%5Cn%20%20AND%20%5C%5C%5C%22event_name%5C%5C%5C%22%20IN%20(%27ads_show%27)%5C%5CnGROUP%20BY%20TIME_FLOOR(CAST(%5C%5C%5C%22__time%5C%5 │
    │ C%5C%22%20AS%20TIMESTAMP),%20%27PT5M%27)%5C%5Cn–ORDER%20BY%20%5C%5C%5C%22HLLSketchMerge%5C%5C%5C%22%20DESC%5C%5CnLIMIT%2050000%5C%22,%5C%22intervals%5C%22:%7B%5C%22type%5C%22:%5C%22intervals%5C%22,%5C%22intervals%5C%22:%5B%5C%22$%7B__from:date:iso%7 │
    │ D%2F$%7B__to:date:iso%7D%5C%22%5D%7D%7D,%5C%22settings%5C%22:%7B%7D%7D%22,%22intervalMs%22:1000%7D%7D,%7B%22refId%22:%22B%22,%22datasourceUid%22:%22-100%22,%22queryType%22:%22%22,%22model%22:%7B%22refId%22:%22B%22,%22hide%22:false,%22type%22:%22class │
    │ ic_conditions%22,%22datasource%22:%7B%22uid%22:%22-100%22,%22type%22:%22__expr__%22%7D,%22conditions%22:%5B%7B%22type%22:%22query%22,%22evaluator%22:%7B%22params%22:%5B3%5D,%22type%22:%22gt%22%7D,%22operator%22:%7B%22type%22:%22and%22%7D,%22query%22: │
    │ %7B%22params%22:%5B%22A%22%5D%7D,%22reducer%22:%7B%22params%22:%5B%5D,%22type%22:%22last%22%7D%7D%5D%7D%7D%5D,%22name%22:%22Ads%20Show%22,%22condition%22:%22B%22,%22annotations%22:%5B%7B%22key%22:%22__dashboardUid__%22,%22value%22:%22g9ZvGt-nz%22%7D, │
    │ %7B%22key%22:%22__panelId__%22,%22value%22:%224%22%7D%5D%7D&returnTo=%2Fd%2Fg9ZvGt-nz%2Fgame-activity-v2%3ForgId%3D1%26editPanel%3D4%26tab%3Dalert”

  • Did you follow any online instructions? If so, what is the URL?