fix(#2954): data type change in adapter causes data storage failure i…

…n influxdb (#2958) * fix(#2954): Add test to reproduce problem * fix(#2954): Add warning message to data type change if adapter is edited
apache · Jun 26, 2024 · 8e49cbc · 8e49cbc
1 parent cfb5882
commit 8e49cbc
Show file tree

Hide file tree

Showing 18 changed files with 179 additions and 37 deletions.
diff --git a/ui/cypress/support/utils/connect/ConnectBtns.ts b/ui/cypress/support/utils/connect/ConnectBtns.ts
@@ -40,6 +40,18 @@ export class ConnectBtns {
         return cy.dataCy('store-edit-adapter');
     }
 
+    public static changeRuntimeType() {
+        return cy.dataCy('connect-change-runtime-type', { timeout: 10000 });
+    }
+
+    public static updateAndMigratePipelines() {
+        return cy.dataCy('btn-update-adapter-migrate-pipelines');
+    }
+
+    public static nextBtn() {
+        return cy.get('button').contains('Next').parent();
+    }
+
     // =====================  Adapter settings btns  ==========================
     public static adapterSettingsStartAdapter() {
         return cy.dataCy('adapter-settings-start-adapter-btn');

diff --git a/.../support/utils/ConnectEventSchemaUtils.ts → .../utils/connect/ConnectEventSchemaUtils.ts b/.../support/utils/ConnectEventSchemaUtils.ts → .../utils/connect/ConnectEventSchemaUtils.ts
@@ -1,21 +1,23 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  *
  */
 
+import { ConnectBtns } from './ConnectBtns';
+
 export class ConnectEventSchemaUtils {
     public static markPropertyAsDimension(propertyName: string) {
         cy.dataCy('property-scope-' + propertyName, { timeout: 10000 })
@@ -196,9 +198,13 @@ export class ConnectEventSchemaUtils {
     public static changePropertyDataType(
         propertyName: string,
         dataType: string,
+        warningIsShown: boolean = false,
     ) {
         ConnectEventSchemaUtils.clickEditProperty(propertyName);
-        cy.dataCy('connect-change-runtime-type')
+
+        this.checkIfWarningIsShown(warningIsShown);
+
+        ConnectBtns.changeRuntimeType()
             .click()
             .get('mat-option')
             .contains(dataType)
@@ -208,13 +214,19 @@ export class ConnectEventSchemaUtils {
         cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click({
             force: true,
         });
-        cy.dataCy('connect-change-runtime-type', { timeout: 10000 }).contains(
-            dataType,
-        );
+        ConnectBtns.changeRuntimeType().contains(dataType);
         cy.wait(1000);
         cy.dataCy('sp-save-edit-property').click();
     }
 
+    private static checkIfWarningIsShown(warningIsShown: boolean) {
+        if (warningIsShown) {
+            cy.dataCy('warning-change-data-type').should('be.visible');
+        } else {
+            cy.dataCy('warning-change-data-type').should('not.exist');
+        }
+    }
+
     public static eventSchemaNextBtnDisabled() {
         cy.get('#event-schema-next-button').should('be.disabled');
     }

diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts
@@ -18,7 +18,7 @@
 
 import { StaticPropertyUtils } from '../StaticPropertyUtils';
 import { AdapterInput } from '../../model/AdapterInput';
-import { ConnectEventSchemaUtils } from '../ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from './ConnectEventSchemaUtils';
 import { DataLakeUtils } from '../datalake/DataLakeUtils';
 import { ConnectBtns } from './ConnectBtns';
 import { AdapterBuilder } from '../../builder/AdapterBuilder';
@@ -208,6 +208,14 @@ export class ConnectUtils {
         );
     }
 
+    public static storeAndStartEditedAdapter() {
+        ConnectUtils.finishEventSchemaConfiguration();
+        ConnectBtns.storeEditAdapter().click();
+        ConnectBtns.updateAndMigratePipelines().click();
+        ConnectUtils.closeAdapterPreview();
+        ConnectBtns.startAdapter().click();
+    }
+
     public static deleteAdapterAndAssociatedPipelines(switchUserCheck = false) {
         // Delete adapter and associated pipelines
         this.goToConnect();

diff --git a/ui/cypress/support/utils/datalake/DataLakeBtns.ts b/ui/cypress/support/utils/datalake/DataLakeBtns.ts
@@ -0,0 +1,23 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+export class DataLakeBtns {
+    public static refreshDataLakeMeasures() {
+        return cy.dataCy('refresh-data-lake-measures');
+    }
+}
diff --git a/ui/cypress/support/utils/datalake/DataLakeUtils.ts b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
@@ -332,4 +332,21 @@ export class DataLakeUtils {
 
         return currentDate;
     }
+
+    public static waitForCountingResults() {
+        cy.dataCy('datalake-number-of-events-spinner', {
+            timeout: 10000,
+        }).should('exist');
+        cy.dataCy('datalake-number-of-events-spinner', {
+            timeout: 10000,
+        }).should('not.exist');
+    }
+
+    public static getDatalakeNumberOfEvents(): Cypress.Chainable<string> {
+        return cy
+            .dataCy('datalake-number-of-events', { timeout: 10000 })
+            .should('be.visible')
+            .invoke('text')
+            .then(text => text.trim());
+    }
 }
diff --git a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
@@ -19,15 +19,18 @@
 import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
 import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
 import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
+import { ConnectEventSchemaUtils } from '../../support/utils/connect/ConnectEventSchemaUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
+import { DataLakeBtns } from '../../support/utils/datalake/DataLakeBtns';
 
 describe('Test Edit Adapter', () => {
     beforeEach('Setup Test', () => {
         // To set up test add a stream adapter that can be configured
         cy.initStreamPipesTest();
-        ConnectUtils.addMachineDataSimulator('simulator');
     });
 
     it('Successfully edit adapter', () => {
+        ConnectUtils.addMachineDataSimulator('simulator');
         const newAdapterName = 'Edited Adapter';
 
         ConnectUtils.goToConnect();
@@ -68,4 +71,47 @@ describe('Test Edit Adapter', () => {
         cy.dataCy('adapter-name').contains(newAdapterName);
         cy.get('.sp-dialog-content').contains(newAdapterName);
     });
+
+    it('Successfully edit adapter with persistence pipeline', () => {
+        ConnectUtils.addMachineDataSimulator('simulator', true);
+
+        ConnectUtils.goToConnect();
+
+        // stop adapter and edit adapter
+        ConnectBtns.stopAdapter().click();
+        ConnectBtns.editAdapter().click();
+
+        // change data type of density to integer
+        ConnectBtns.nextBtn().click();
+        ConnectEventSchemaUtils.changePropertyDataType(
+            'density',
+            'Integer',
+            true,
+        );
+
+        ConnectUtils.storeAndStartEditedAdapter();
+
+        // Validate that the data is further persisted in the database by checking if the amount of events in the data lake changes
+        DataLakeUtils.goToDatalakeConfiguration();
+
+        DataLakeUtils.waitForCountingResults();
+
+        let initialValue;
+
+        DataLakeUtils.getDatalakeNumberOfEvents().then(value => {
+            initialValue = value;
+        });
+
+        DataLakeBtns.refreshDataLakeMeasures().click();
+
+        DataLakeUtils.waitForCountingResults();
+
+        DataLakeUtils.getDatalakeNumberOfEvents().then(newValue => {
+            // IMPORTANT: Currently we implemented a workaround by showing the user a warning message when the data type is changed.
+            // In the future, we need a migration mechanism to automatically change all the StreamPipes resources that are effected
+            // by the change. Once this is implemented the following line must be changed to .not.equal.
+            // The issue is tracked here: https://github.com/apache/streampipes/issues/2954
+            expect(newValue).equal(initialValue);
+        });
+    });
 });
diff --git a/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts b/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts
@@ -70,7 +70,7 @@ describe('Test Edit Adapter and Pipeline', () => {
         cy.dataCy('sp-connect-adapter-edit-warning', {
             timeout: 60000,
         }).should('be.visible');
-        cy.dataCy('btn-update-adapter-migrate-pipelines').click();
+        ConnectBtns.updateAndMigratePipelines().click();
         ConnectUtils.closeAdapterPreview();
         cy.wait(1000);
 

diff --git a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
@@ -50,7 +50,7 @@ describe('Test Edit Adapter', () => {
             .clear()
             .type('http://schema.org/Numbers');
         // Change field data type
-        cy.dataCy('connect-change-runtime-type')
+        ConnectBtns.changeRuntimeType()
             .click()
             .get('mat-option')
             .contains('Double')
@@ -83,10 +83,7 @@ describe('Test Edit Adapter', () => {
             'have.value',
             'http://schema.org/Numbers',
         );
-        cy.dataCy('connect-change-runtime-type').should(
-            'include.text',
-            'Double',
-        );
+        ConnectBtns.changeRuntimeType().should('include.text', 'Double');
         cy.dataCy('connect-schema-correction-value').should('have.value', '2');
         cy.dataCy('connect-schema-correction-operator').should(
             'include.text',
@@ -96,7 +93,7 @@ describe('Test Edit Adapter', () => {
         // Delete inserted values in edit field
         cy.dataCy('connect-edit-field-runtime-name').clear();
         cy.get('[id="domainproperty"]').clear();
-        cy.dataCy('connect-change-runtime-type')
+        ConnectBtns.changeRuntimeType()
             .click()
             .get('mat-option')
             .contains('Float')

diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
@@ -17,7 +17,7 @@
  */
 import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
 import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils';
 import { AdapterBuilder } from '../../../support/builder/AdapterBuilder';
 import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns';
 

diff --git a/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts b/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts
@@ -18,7 +18,7 @@
 
 import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
 import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils';
 
 describe('Connect schema rule transformations', () => {
     beforeEach('Setup Test', () => {

diff --git a/ui/cypress/tests/adapter/rules/streamRules.spec.ts b/ui/cypress/tests/adapter/rules/streamRules.spec.ts
@@ -18,7 +18,7 @@
 
 import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
 import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils';
 
 describe('Connect aggregation rule transformations', () => {
     beforeEach('Setup Test', () => {

diff --git a/ui/cypress/tests/adapter/rules/valueRules.ts b/ui/cypress/tests/adapter/rules/valueRules.ts
@@ -18,7 +18,7 @@
 
 import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
 import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils';
 
 describe('Connect value rule transformations', () => {
     beforeEach('Setup Test', () => {

diff --git a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
@@ -36,7 +36,8 @@
                             <button
                                 color="accent"
                                 mat-icon-button
-                                matTooltip="Refresh Schema"
+                                matTooltip="Refresh"
+                                data-cy="refresh-data-lake-measures"
                                 (click)="loadAvailableMeasurements()"
                             >
                                 <mat-icon>refresh</mat-icon>
@@ -97,6 +98,7 @@ <h4 style="margin-bottom: 0px">
                                     fxLayoutAlign="center"
                                     style="margin: 10px 0 5px 0"
                                     color="accent"
+                                    data-cy="datalake-number-of-events-spinner"
                                     *ngIf="configurationEntry.events < 0"
                                     >Loading
                                 </mat-spinner>

diff --git a/...-property/components/edit-schema-transformation/edit-schema-transformation.component.html b/...-property/components/edit-schema-transformation/edit-schema-transformation.component.html
@@ -92,6 +92,21 @@
         sectionDescription="The data type of the field values"
         *ngIf="!isTimestampProperty && !isNestedProperty"
     >
+        <div *ngIf="adapterIsInEditMode" class="warning-message">
+            <p data-cy="warning-change-data-type" class="text-center">
+                Changing the data type can result in pipelines no longer writing
+                the data correctly to the data lake.
+                <br />
+                <br />
+                To resolve this problem, you can:
+                <br />
+                - Rename the property.
+                <br />
+                - Truncate the old data.
+                <br />
+                - Write the data to a new measurement.
+            </p>
+        </div>
         <sp-edit-data-type
             [cachedProperty]="cachedProperty"
             (dataTypeChanged)="dataTypeChanged.emit($event)"

diff --git a/...-property/components/edit-schema-transformation/edit-schema-transformation.component.scss b/...-property/components/edit-schema-transformation/edit-schema-transformation.component.scss
@@ -15,3 +15,5 @@
  * limitations under the License.
  *
  */
+
+@import '../../edit-event-property.component.scss';
diff --git a/...nt-property/components/edit-schema-transformation/edit-schema-transformation.component.ts b/...nt-property/components/edit-schema-transformation/edit-schema-transformation.component.ts
@@ -31,6 +31,7 @@ import {
     SemanticType,
     SemanticTypesRestService,
 } from '@streampipes/platform-services';
+import { Router } from '@angular/router';
 
 @Component({
     selector: 'sp-edit-schema-transformation',
@@ -52,9 +53,12 @@ export class EditSchemaTransformationComponent implements OnInit {
     domainPropertyControl = new UntypedFormControl();
     semanticTypes: Observable<string[]>;
 
+    adapterIsInEditMode: boolean;
+
     constructor(
         private semanticTypesRestService: SemanticTypesRestService,
         private shepherdService: ShepherdService,
+        private router: Router,
     ) {}
 
     ngOnInit(): void {
@@ -68,6 +72,8 @@ export class EditSchemaTransformationComponent implements OnInit {
                     : [];
             }),
         );
+
+        this.adapterIsInEditMode = this.router.url.includes('connect/edit');
     }
 
     editTimestampDomainProperty(checked: boolean) {

diff --git a/...ents/edit-value-transformation/edit-correction-value/edit-correction-value.component.scss b/...ents/edit-value-transformation/edit-correction-value/edit-correction-value.component.scss
@@ -16,10 +16,4 @@
  *
  */
 
-.warning-message {
-    margin-bottom: 10px;
-    padding: 5px;
-    border: 1px solid var(--color-warn);
-    background: var(--color-bg-1);
-    font-size: 14px;
-}
+@import '../../../edit-event-property.component.scss';
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,3 +15,5 @@ @@
      * limitations under the License.
      *
      */
+    @import '../../edit-event-property.component.scss';